From 1ddcffa32d2016015b43b30f503321a878c48285 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 22:33:39 +0200 Subject: [PATCH 01/72] build: zip for macos previsouly zip was built before pkg (as makefile dependency) since pkg's only depend on rebar3 release tarball (not .zip) makefile was updated but forgot to update the pkg build for macos --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index af7ff9865..4d2997b84 100755 --- a/build +++ b/build @@ -134,7 +134,7 @@ case "$ARTIFACT" in ;; pkg) if [ -z "${PKGERDIR:-}" ]; then - # zip should have been built as a Makefile dependency + make_zip log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi From 934d3c405d603bcbae5b375db92839c7ac59817d Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 11:54:40 +0800 Subject: [PATCH 02/72] chore(CI): update build workflows --- .ci/build_packages/Dockerfile | 2 + .github/workflows/build_packages.yaml | 10 +-- .github/workflows/build_slim_packages.yaml | 73 +++++++++++++++++++++- build | 1 - 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index aa45c970b..792acddf5 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -7,6 +7,8 @@ COPY . /emqx WORKDIR /emqx +RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump + RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump RUN /emqx/.ci/build_packages/tests.sh diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 697820299..f1ddef88a 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -123,6 +123,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + erl_otp: + - 23.2.7.2 exclude: - profile: emqx-edge @@ -141,12 +143,12 @@ jobs: - name: build erlang timeout-minutes: 60 run: | - kerl build 23.2.7 - kerl install 23.2.7 $HOME/.kerl/23.2.7 + kerl build ${{ matrix.erl_otp }} + kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build run: | - . $HOME/.kerl/23.2.7/activate - make -C source ${{ matrix.profile }}-pkg + . $HOME/.kerl/${{ matrix.erl_otp }}/activate + make -C source ${{ matrix.profile }}-zip - name: test run: | cd source diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a78d93a56..aae5cfd18 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,8 +1,12 @@ name: Build slim packages on: - - pull_request - - workflow_dispatch + push: + tags: + - v* + - e* + pull_request: + workflow_dispatch: jobs: build: @@ -30,7 +34,9 @@ jobs: else echo "EMQX_NAME=emqx" >> $GITHUB_ENV fi - - name: build packages + - name: build zip packages + run: make ${EMQX_NAME}-zip + - name: build deb/rpm packages run: make ${EMQX_NAME}-pkg - name: pakcages test run: | @@ -40,3 +46,64 @@ jobs: with: name: ${{ matrix.os }} path: _packages/**/*.zip + + mac: + runs-on: macos-10.15 + + strategy: + matrix: + erl_otp: + - 23.2.7.2 + + steps: + - uses: actions/checkout@v1 + - name: prepare + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token + echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV + else + echo "EMQX_NAME=emqx" >> $GITHUB_ENV + fi + - name: prepare + run: | + brew install curl zip unzip gnu-sed kerl unixodbc freetds + echo "/usr/local/bin" >> $GITHUB_PATH + git config --global credential.helper store + - name: build erlang + timeout-minutes: 60 + run: | + kerl build ${{ matrix.erl_otp }} + kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + - name: build + run: | + . $HOME/.kerl/${{ matrix.erl_otp }}/activate + make ${EMQX_NAME}-zip + - name: test + run: | + pkg_name=$(basename _packages/emqx/emqx-*.zip) + unzip _packages/emqx/$pkg_name + gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 + ready='no' + for i in {1..10}; do + if curl -fs 127.0.0.1:18083 > /dev/null; then + ready='yes' + break + fi + sleep 1 + done + if [ "$ready" != "yes" ]; then + echo "Timed out waiting for emqx to be ready" + cat emqx/log/erlang.log.1 + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx stop + rm -rf emqx + - uses: actions/upload-artifact@v1 + with: + name: macos + path: _packages/emqx/. diff --git a/build b/build index 4d2997b84..be7813e66 100755 --- a/build +++ b/build @@ -134,7 +134,6 @@ case "$ARTIFACT" in ;; pkg) if [ -z "${PKGERDIR:-}" ]; then - make_zip log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi From c5aeeb3b40b79abeee8f6a3a9cff9538ff94bac0 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 14:11:49 +0800 Subject: [PATCH 03/72] fix(import): fix import bridge mqtt test cases --- ...x_bridge_mqtt_data_export_import_SUITE.erl | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index 2609e1006..7cf7d19cf 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -28,24 +28,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - ok = ekka_mnesia:start(), - ok = emqx_rule_registry:mnesia(boot), - ok = emqx_rule_engine:load_providers(), - emqx_ct_helpers:start_apps([emqx_web_hook, - emqx_bridge_mqtt, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard]), + emqx_ct_helpers:start_apps([emqx_bridge_mqtt, emqx_rule_engine]), Cfg. end_per_suite(Cfg) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, - emqx_management, - emqx_modules, - emqx_rule_engine, - emqx_bridge_mqtt, - emqx_web_hook]), + emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. get_data_path() -> @@ -53,8 +40,8 @@ get_data_path() -> import(FilePath, Version) -> ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + timer:sleep(500), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> - timer:sleep(2000), case Id of <<"bridge">> -> test_utils:resource_is_alive(Id), @@ -74,19 +61,23 @@ import(FilePath, Version) -> t_import420(_) -> import("420.json", 420), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import430(_) -> import("430.json", 430), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import409(_) -> import("409.json", 409), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import415(_) -> import("415.json", 415), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). handle_config(Config, 420, bridge) -> @@ -126,27 +117,33 @@ handle_config(_, _, _) -> ok. t_importee4010(_) -> import("ee4010.json", ee4010), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee410(_) -> import("ee410.json", ee410), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee411(_) -> import("ee411.json", ee411), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee420(_) -> import("ee420.json", ee420), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee425(_) -> import("ee425.json", ee425), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee430(_) -> import("ee430.json", ee430), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). %%-------------------------------------------------------------------- %% handle_config @@ -176,3 +173,11 @@ handle_config(Config, ee430, rpc) -> handle_config(Config, ee435, Id) -> handle_config(Config, ee430, Id). -endif. + +remove_resources() -> + timer:sleep(500), + lists:foreach(fun(#resource{id = Id}) -> + emqx_rule_registry:remove_resource(Id), + emqx_rule_registry:remove_resource_params(Id) + end, emqx_rule_registry:get_resources()), + timer:sleep(500). \ No newline at end of file From 80ef076ceec35857bffec62d146038ceac656ece Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 16:32:17 +0800 Subject: [PATCH 04/72] feat(rule_engine): rule sql add null function --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 4 ++++ apps/emqx_rule_engine/src/emqx_rule_utils.erl | 1 + 2 files changed, 5 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 4f8851d40..a96ee7a62 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -35,6 +35,7 @@ , contains_topic/3 , contains_topic_match/2 , contains_topic_match/3 + , null/0 ]). %% Arithmetic Funcs @@ -300,6 +301,9 @@ find_topic_filter(Filter, TopicFilters, Func) -> throw:Result -> Result end. +null() -> + undefined. + %%------------------------------------------------------------------------------ %% Arithmetic Funcs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index ff7ee0304..3791b1386 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -228,6 +228,7 @@ tcp_connectivity(Host, Port, Timeout) -> unwrap(<<"${", Val/binary>>) -> binary:part(Val, {0, byte_size(Val)-1}). +sql_data(undefined) -> null; sql_data(List) when is_list(List) -> List; sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; From a07fece436ec65026d8d0b0cd531de08ad408962 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 20:47:46 +0800 Subject: [PATCH 05/72] chore(slim): fix build silm macosx fail --- .github/workflows/build_slim_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index aae5cfd18..f3362cd44 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -83,8 +83,8 @@ jobs: make ${EMQX_NAME}-zip - name: test run: | - pkg_name=$(basename _packages/emqx/emqx-*.zip) - unzip _packages/emqx/$pkg_name + pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) + unzip _packages/${EMQX_NAME}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' From 29bbc0802bc146143f4d373a38282e4b65c9eb37 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 13:59:24 +0800 Subject: [PATCH 06/72] chore(CI): use cache for mac --- .github/workflows/build_packages.yaml | 9 ++++++++- .github/workflows/build_slim_packages.yaml | 13 ++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f1ddef88a..fde6795b1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -137,10 +137,17 @@ jobs: run: unzip -q source.zip - name: prepare run: | + brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store + - uses: actions/cache@v2 + id: cache + with: + path: ~/.kerl + key: erl${{ matrix.erl_otp }}-macos10.15 - name: build erlang + if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 run: | kerl build ${{ matrix.erl_otp }} @@ -153,7 +160,7 @@ jobs: run: | cd source pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip _packages/${{ matrix.profile }}/$pkg_name + unzip -q _packages/${{ matrix.profile }}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index f3362cd44..573b84e1b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -69,10 +69,17 @@ jobs: fi - name: prepare run: | + brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store + - uses: actions/cache@v2 + id: cache + with: + path: ~/.kerl + key: erl${{ matrix.erl_otp }}-macos10.15 - name: build erlang + if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 run: | kerl build ${{ matrix.erl_otp }} @@ -84,7 +91,7 @@ jobs: - name: test run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip _packages/${EMQX_NAME}/$pkg_name + unzip -q _packages/${EMQX_NAME}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -103,7 +110,7 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v2 with: name: macos - path: _packages/emqx/. + path: _packages/**/*.zip From 1cabd140c5701c3ee1b503d421c30c21f7312adb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 11:11:56 +0200 Subject: [PATCH 07/72] fix(logging): add line-break after each JSON log --- src/emqx.app.src | 2 +- src/emqx.appup.src | 7 ++++++- src/emqx_logger_jsonfmt.erl | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index a9ae81cac..449ffd311 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 87ca0c005..5b6a5a4d5 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,9 +1,14 @@ %% -*-: erlang -*- {VSN, - [ + [ {"4.3.0", + [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ], [ + {"4.3.0", + [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ] }. diff --git a/src/emqx_logger_jsonfmt.erl b/src/emqx_logger_jsonfmt.erl index 7d0a1a328..31e4c2fda 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/src/emqx_logger_jsonfmt.erl @@ -48,7 +48,7 @@ -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) -> Config = add_default_config(Config0), - format(Msg, Meta#{level => Level}, Config). + [format(Msg, Meta#{level => Level}, Config) , "\n"]. format(Msg, Meta, Config) -> Data0 = From 3aba50605adc6559ca222b6fe5e43919d0545cba Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 16:05:51 +0200 Subject: [PATCH 08/72] chore(nodetool): ensure correct lib dirs Prioro to this change, ekka lib dir was added by inspecting the ekka lib version in persisted RELEASES file. However, this file may not be the true version if a hot upgrade failed in the middle. In this fix, it reads the per-version emqx.rel file content to load lib paths. --- bin/emqx | 3 ++- bin/nodetool | 20 ++++++++++++++++++++ data/emqx_vars | 1 - 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index da1ac8990..fd6986614 100755 --- a/bin/emqx +++ b/bin/emqx @@ -174,9 +174,10 @@ relx_gen_id() { # Control a node relx_nodetool() { command="$1"; shift + export RUNNER_ROOT_DIR + export REL_VSN ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \ - ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ -setcookie "$COOKIE" "$command" "$@" } diff --git a/bin/nodetool b/bin/nodetool index e67ef4790..5ab5ec69a 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -19,6 +19,7 @@ main(Args) -> ok end end, + ok = add_libs_dir(), ok = do_with_halt(Args, "mnesia_dir", fun create_mnesia_dir/2), ok = do_with_halt(Args, "chkconfig", fun("-config", X) -> chkconfig(X) end), ok = do_with_halt(Args, "chkconfig", fun chkconfig/1), @@ -288,3 +289,22 @@ join([], Sep) when is_list(Sep) -> []; join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). + +add_libs_dir() -> + [_ | _] = RootDir = os:getenv("RUNNER_ROOT_DIR"), + RelFile = filename:join([RootDir, "releases", + os:getenv("REL_VSN"), + "emqx.rel" + ]), + {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} = file:consult(RelFile), + lists:foreach( + fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); + ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) + end, Libs). + +add_lib_dir(RootDir, Name, Vsn) -> + LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), + case code:add_pathz(LibDir) of + true -> ok; + {error, _} -> error(LibDir) + end. diff --git a/data/emqx_vars b/data/emqx_vars index 8081badbe..453bd618f 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -13,7 +13,6 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" -LIB_EKKA_DIR="${RUNNER_LIB_DIR}/ekka-$(grep ekka "${RUNNER_ROOT_DIR}/releases/RELEASES" | awk -F '\"' '{print $2}')" EMQX_LICENSE_CONF='' export EMQX_DESCRIPTION='{{ emqx_description }}' From 50f4e0617bc97d900cae06d008d1b7c5f8ea5b3e Mon Sep 17 00:00:00 2001 From: z8674558 Date: Mon, 10 May 2021 15:08:36 +0900 Subject: [PATCH 09/72] fix(docker-entrypoint): do not mv loaded_plugins file, copy it by cat --- .ci/docker-compose-file/conf.cluster.env | 1 + .github/workflows/run_fvt_tests.yaml | 7 +++++++ deploy/docker/docker-entrypoint.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/conf.cluster.env b/.ci/docker-compose-file/conf.cluster.env index 16fb4733e..d8294a785 100644 --- a/.ci/docker-compose-file/conf.cluster.env +++ b/.ci/docker-compose-file/conf.cluster.env @@ -4,3 +4,4 @@ EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io" EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on EMQX_LOG__LEVEL=debug +EMQX_LOADED_PLUGINS=emqx_sn diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 68aec59a6..8f65d022e 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -46,6 +46,13 @@ jobs: echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; sleep 5; done + - name: verify EMQX_LOADED_PLUGINS override working + run: | + expected="{emqx_sn, true}." + output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) + if [ "$expected" != "$output" ]; then + exit 1 + fi - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index a95aa50ee..16b6cb077 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -98,7 +98,7 @@ fill_tuples() { local elements=${*:2} for var in $elements; do if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then - sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && mv tmpfile "$file" + sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" elif grep -q "$var\s*\." "$file"; then # backward compatible. sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" From 7afd4940e7f96b9d3cb52d32f2408fc140131b4b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 12:10:21 +0200 Subject: [PATCH 10/72] chore(config): do not expose manual rpc peer port config when using `manual`, all nodes in the cluster should agree to the same port number, so there is no need to make peer port number for clients. --- etc/emqx.conf | 17 ++++++----------- priv/emqx.schema | 16 +++++++++------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index ef1769a9e..bfd9674e8 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -331,25 +331,20 @@ rpc.async_batch_size = 256 ## Defaults to `stateless`. rpc.port_discovery = stateless -## TCP server port for RPC. +## TCP port number for RPC server to listen on. ## ## Only takes effect when `rpc.port_discovery` = `manual`. ## +## NOTE: All nodes in the cluster should agree to this same config. +## ## Value: Port [1024-65535] #rpc.tcp_server_port = 5369 -## TCP port for outgoing RPC connections. -## -## Only takes effect when `rpc.port_discovery` = `manual`. -## -## Value: Port [1024-65535] -#rpc.tcp_client_port = 5369 - ## Number of outgoing RPC connections. ## -## Value: Interger [1-256] -## Defaults to NumberOfCPUSchedulers / 2 -#rpc.tcp_client_num = 1 +## Value: Interger [0-256] +## Defaults to NumberOfCPUSchedulers / 2 when set to 0 +#rpc.tcp_client_num = 0 ## RCP Client connect timeout. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 5ecd62de0..aee2fddc6 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -367,13 +367,7 @@ end}. {datatype, integer} ]}. -%% Default TCP port for outgoing connections -{mapping, "rpc.tcp_client_port", "gen_rpc.tcp_client_port", [ - {default, 5369}, - {datatype, integer} -]}. - -%% Default TCP port for outgoing connections +%% Number of tcp connections when connecting to RPC server {mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ {default, 0}, {datatype, integer}, @@ -451,6 +445,14 @@ end}. fun(X) -> X >= 0 andalso X < 256 end }. +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +{translation, "gen_rpc.tcp_client_port", fun(_, _, Conf) -> + cuttlefish:conf_get("rpc.tcp_server_port", Conf) +end}. + %%-------------------------------------------------------------------- %% Log %%-------------------------------------------------------------------- From bb490e7d6e76a35631af458475b879c6ac8282a7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 14:43:14 +0800 Subject: [PATCH 11/72] chore(CI): add apps version check --- .github/workflows/apps_version_check.yaml | 12 ++++++++ scripts/apps-version-check.sh | 34 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/workflows/apps_version_check.yaml create mode 100755 scripts/apps-version-check.sh diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml new file mode 100644 index 000000000..808e82600 --- /dev/null +++ b/.github/workflows/apps_version_check.yaml @@ -0,0 +1,12 @@ +name: Check Apps Version + +on: [pull_request] + +jobs: + check_deps_integrity: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v1 + - name: Check apps version + run: ./scripts/apps-version-check.sh diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh new file mode 100755 index 000000000..94b3c775e --- /dev/null +++ b/scripts/apps-version-check.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1)") + +bad_app_count=0 + +while read -r app; do + if [ "$app" != "emqx" ]; then + app_path="$app" + else + app_path="." + fi + src_file="$app_path/src/$(basename "$app").app.src" + old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" + now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') + if [ "$old_app_version" = "$now_app_version" ]; then + changed="$(git diff --name-only "$latest_release"...HEAD \ + -- "$app_path/etc" \ + -- "$app_path/src" \ + -- "$app_path/priv" \ + -- "$app_path/c_src" | wc -l)" + if [ "$changed" -gt 0 ]; then + echo "$src_file needs a vsn bump" + bad_app_count=$(( bad_app_count + 1)) + fi + fi +done < <(./scripts/find-apps.sh) + +if [ $bad_app_count -gt 0 ]; then + exit 1 +else + echo "apps version check successfully" +fi From 873459a07fd8b4046397b38a36f196cb55c71370 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 11 May 2021 14:03:26 +0200 Subject: [PATCH 12/72] fix(emqx_ctl): ensure env variables are exported --- bin/emqx_ctl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 73f2fc960..4c3aaa1ca 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -8,6 +8,9 @@ ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$ROOT_DIR"/releases/emqx_vars +export RUNNER_ROOT_DIR +export REL_VSN + # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" if [ -z "$LATEST_VM_ARGS" ]; then From ded494f5444101e9a0fdc74b57acd3e690b2e12b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 11 May 2021 14:20:14 +0200 Subject: [PATCH 13/72] fix(emqx_rule_engine): add appup --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_rule_engine/src/emqx_rule_engine.appup.src diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 14483b9b2..86846ccde 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src new file mode 100644 index 000000000..17cecac68 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -0,0 +1,14 @@ +%% -*-: erlang -*- +{"4.3.1", + [ {"4.3.0", + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. From 0bf76adfd56f46330b4e1515c2afb29d3783ae96 Mon Sep 17 00:00:00 2001 From: Rory-Z Date: Wed, 12 May 2021 01:19:07 +0000 Subject: [PATCH 14/72] chore(CI): fix app version check --- .github/workflows/apps_version_check.yaml | 2 +- scripts/apps-version-check.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 808e82600..4975e5c11 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -3,7 +3,7 @@ name: Check Apps Version on: [pull_request] jobs: - check_deps_integrity: + check_apps_version: runs-on: ubuntu-20.04 steps: diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 94b3c775e..49350b007 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1)") +latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1 --remotes=refs/remote/origin)") bad_app_count=0 From 09fb1461b711b60c0e0019e3274f5c4aaa9abdb3 Mon Sep 17 00:00:00 2001 From: wwhai Date: Wed, 12 May 2021 22:13:26 +0800 Subject: [PATCH 15/72] fix(mgmt): fix diylazer warning for function return (#4786) --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_management.appup.src | 12 ++++++++++++ apps/emqx_management/src/emqx_mgmt_data_backup.erl | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_management/src/emqx_management.appup.src diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index b1e04c439..fe65052cf 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src new file mode 100644 index 000000000..5048e4f0f --- /dev/null +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -0,0 +1,12 @@ +%% -*-: erlang -*- +{"4.3.1", + [ {"4.3.0", + [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + ]} + ], + [ + {"4.3.0", + [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + ]} + ] +}. \ No newline at end of file diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index c9df698ed..1eb6f8245 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -514,7 +514,7 @@ import_modules(Modules) -> <<"enabled">> := Enabled, <<"created_at">> := CreatedAt, <<"description">> := Description}) -> - emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) end, Modules) end. From 1ba8d71e7e21e2ddd46000ccc3f66d883927d880 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:10:49 +0800 Subject: [PATCH 16/72] fix(metrics): set the retained/delayed metrics to counter type The metrics for messages.* shows the number of times such messages are received, and it should be a counter rather than a gauge type. --- src/emqx.appup.src | 18 ++++++++++++------ src/emqx_metrics.erl | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 5b6a5a4d5..b10c14a9c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,14 +1,20 @@ %% -*-: erlang -*- {VSN, - [ {"4.3.0", - [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} - ]}, + [ + {"4.3.0", [ + {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + {load_module, emqx_metrics, brutal_purge, soft_purge, []}, + {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} + ]}, {<<".*">>, []} ], [ - {"4.3.0", - [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} - ]}, + {"4.3.0", [ + {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + %% Just load the module. We don't need to change the 'messages.retained' + %% and 'messages.retained' counter type. + {load_module, emqx_metrics, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ] }. diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 44009d4e2..3b96bfbad 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -65,6 +65,10 @@ , code_change/3 ]). +%% BACKW: v4.3.0 +-export([ upgrade_retained_delayed_counter_type/0 + ]). + -export_type([metric_idx/0]). -compile({inline, [inc/1, inc/2, dec/1, dec/2]}). @@ -145,8 +149,8 @@ {counter, 'messages.dropped.expired'}, % QoS2 Messages expired {counter, 'messages.dropped.no_subscribers'}, % Messages dropped {counter, 'messages.forward'}, % Messages forward - {gauge, 'messages.retained'}, % Messages retained - {gauge, 'messages.delayed'}, % Messages delayed + {counter, 'messages.retained'}, % Messages retained + {counter, 'messages.delayed'}, % Messages delayed {counter, 'messages.delivered'}, % Messages delivered {counter, 'messages.acked'} % Messages acked ]). @@ -195,6 +199,19 @@ start_link() -> -spec(stop() -> ok). stop() -> gen_server:stop(?SERVER). +%% BACKW: v4.3.0 +upgrade_retained_delayed_counter_type() -> + case ets:info(?TAB, name) of + ?TAB -> + [M1] = ets:lookup(?TAB, 'messages.retained'), + [M2] = ets:lookup(?TAB, 'messages.delayed'), + true = ets:insert(?TAB, M1#metric{type = counter}), + true = ets:insert(?TAB, M2#metric{type = counter}), + ok; + _ -> + ok + end. + %%-------------------------------------------------------------------- %% Metrics API %%-------------------------------------------------------------------- From e7252eee815eda34a311eca88ae6516ba1992e14 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:16:21 +0800 Subject: [PATCH 17/72] fix(metric): incr the messages.retained counter --- apps/emqx_retainer/src/emqx_retainer.appup.src | 15 +++++++++++++++ apps/emqx_retainer/src/emqx_retainer.erl | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_retainer/src/emqx_retainer.appup.src diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src new file mode 100644 index 000000000..a17e6ee2f --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {load_module, emqx_retainer, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {load_module, emqx_retainer, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index c1523b7a1..340e6929d 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -200,7 +200,7 @@ sort_retained(Msgs) -> store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> case {is_table_full(Env), is_too_big(size(Payload), Env)} of {false, false} -> - ok = emqx_metrics:set('messages.retained', retained_count()), + ok = emqx_metrics:inc('messages.retained'), mnesia:dirty_write(?TAB, #retained{topic = topic2tokens(Topic), msg = Msg, expiry_time = get_expiry_time(Msg, Env)}); From 82330ffa00cbaaa7c739ece3ed894ae5c7f9ef4e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:18:31 +0800 Subject: [PATCH 18/72] feat(delayed): record the delayed table size by emqx_stats There are three indictors about delayed module: - 'messages.delayed': shows the number of delayed msg received - 'emqx_delayed_count': shows the delayed table size now - 'emqx_delayed_max': shows the maximum size that the delayed table has reached --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- lib-ce/emqx_modules/src/emqx_mod_delayed.erl | 39 ++++++++++++++----- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 15 +++++++ 4 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 lib-ce/emqx_modules/src/emqx_modules.appup.src diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index feb18ac2e..c3ffb9f90 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl index 852fc4717..ac5be58b2 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl @@ -22,6 +22,8 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-logger_header("[Delayed]"). + %% Mnesia bootstrap -export([mnesia/1]). @@ -90,7 +92,11 @@ description() -> %% Hooks %%-------------------------------------------------------------------- -on_message_publish(Msg = #message{id = Id, topic = <<"$delayed/", Topic/binary>>, timestamp = Ts}) -> +on_message_publish(Msg = #message{ + id = Id, + topic = <<"$delayed/", Topic/binary>>, + timestamp = Ts + }) -> [Delay, Topic1] = binary:split(Topic, <<"/">>), PubAt = case binary_to_integer(Delay) of Interval when Interval < ?MAX_INTERVAL -> @@ -127,42 +133,57 @@ store(DelayedMsg) -> %%-------------------------------------------------------------------- init([]) -> - {ok, ensure_publish_timer(#{timer => undefined, publish_at => 0})}. + {ok, ensure_stats_event( + ensure_publish_timer(#{timer => undefined, publish_at => 0}))}. handle_call({store, DelayedMsg = #delayed_message{key = Key}}, _From, State) -> ok = mnesia:dirty_write(?TAB, DelayedMsg), - emqx_metrics:set('messages.delayed', delayed_count()), + emqx_metrics:inc('messages.delayed'), {reply, ok, ensure_publish_timer(Key, State)}; handle_call(Req, _From, State) -> - ?LOG(error, "[Delayed] Unexpected call: ~p", [Req]), + ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "[Delayed] Unexpected cast: ~p", [Msg]), + ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. %% Do Publish... handle_info({timeout, TRef, do_publish}, State = #{timer := TRef}) -> DeletedKeys = do_publish(mnesia:dirty_first(?TAB), os:system_time(seconds)), lists:foreach(fun(Key) -> mnesia:dirty_delete(?TAB, Key) end, DeletedKeys), - emqx_metrics:set('messages.delayed', delayed_count()), {noreply, ensure_publish_timer(State#{timer := undefined, publish_at := 0})}; +handle_info(stats, State = #{stats_fun := StatsFun}) -> + StatsFun(delayed_count()), + {noreply, State, hibernate}; + handle_info(Info, State) -> - ?LOG(error, "[Delayed] Unexpected info: ~p", [Info]), + ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, #{timer := TRef}) -> emqx_misc:cancel_timer(TRef). -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +code_change({down, Vsn}, State, _Extra) when Vsn =:= "4.3.0" -> + NState = maps:with([timer, publish_at], State), + {ok, NState}; + +code_change(Vsn, State, _Extra) when Vsn =:= "4.3.0" -> + NState = ensure_stats_event(State), + {ok, NState}. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +%% Ensure the stats +ensure_stats_event(State) -> + StatsFun = emqx_stats:statsfun('delayed.count', 'delayed.max'), + {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), + State#{stats_fun => StatsFun, stats_timer => StatsTimer}. + %% Ensure publish timer ensure_publish_timer(State) -> ensure_publish_timer(mnesia:dirty_first(?TAB), State). diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index a969fd30d..576316703 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.0"}, + {vsn, "4.3.1"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src new file mode 100644 index 000000000..b44a65c17 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {update, emqx_mod_delayed, {advanced, []}} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {update, emqx_mod_delayed, {advanced, []}} + ]}, + {<<".*">>, []} + ] +}. From 36c72583370079dbb3ff241528053ca12eb37387 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 7 May 2021 15:07:06 +0800 Subject: [PATCH 19/72] feat(lwm2m): observe specified object lists --- apps/emqx_lwm2m/etc/emqx_lwm2m.conf | 8 ++++++-- apps/emqx_lwm2m/priv/emqx_lwm2m.schema | 12 ++++++++++-- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 17 ++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf index 152aee604..968b8fd19 100644 --- a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf +++ b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf @@ -13,8 +13,12 @@ lwm2m.lifetime_max = 86400s # the downlink commands sent to the client will be cached. #lwm2m.qmode_time_window = 22 -# Auto send observer command to device -# on | off +# Auto send observer command to device. It can be configured as an OjbectList +# so that emqx will automatically observe the objects in this list. +# +# For examples: "/3/0,/3/0/1,/32976" +# +# Value: off | on | String #lwm2m.auto_observe = off # The topic subscribed by the lwm2m client after it is connected diff --git a/apps/emqx_lwm2m/priv/emqx_lwm2m.schema b/apps/emqx_lwm2m/priv/emqx_lwm2m.schema index b5ed778f5..bf5f144e0 100644 --- a/apps/emqx_lwm2m/priv/emqx_lwm2m.schema +++ b/apps/emqx_lwm2m/priv/emqx_lwm2m.schema @@ -26,8 +26,8 @@ ]}. {mapping, "lwm2m.auto_observe", "emqx_lwm2m.auto_observe", [ - {datatype, flag}, - {default, off} + {datatype, string}, + {default, "off"} %% BACKW: v4.3.0 ]}. {mapping, "lwm2m.lb", "emqx_lwm2m.options", [ @@ -39,6 +39,14 @@ {datatype, bytesize} ]}. +{translation, "emqx_lwm2m.auto_observe", fun(Conf) -> + case cuttlefish:conf_get("lwm2m.auto_observe", Conf, "off") of + "off" -> false; %% BACKW: v4.3.0 + "on" -> true; %% BACKW: v4.3.0 + Str -> string:tokens(Str, ", ") + end +end}. + {translation, "emqx_lwm2m.bind_udp", fun(Conf) -> Options = cuttlefish_variable:filter_by_prefix("lwm2m.bind.udp", Conf), lists:map(fun({_, Bind}) -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index bcc2267cd..55f992da6 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -275,13 +275,24 @@ do_send_to_broker(EventType, Payload, Lwm2mState) -> %% Auto Observe %%-------------------------------------------------------------------- +auto_observe_object_list(true = _Expected, Registered) -> + Registered; +auto_observe_object_list(Expected, Registered) -> + Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), + lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). + send_auto_observe(CoapPid, RegInfo) -> %% - auto observe the objects case proplists:get_value(auto_observe, lwm2m_coap_responder:options(), false) of - true -> + false -> + ?LOG(info, "Auto Observe Disabled", []); + TrueOrObjList -> + Objectlists = auto_observe_object_list( + TrueOrObjList, + maps:get(<<"objectList">>, RegInfo, []) + ), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, maps:get(<<"objectList">>, RegInfo, []), CoapPid); - _ -> ?LOG(info, "Auto Observe Disabled", []) + auto_observe(AlternatePath, Objectlists, CoapPid) end. auto_observe(AlternatePath, ObjectList, CoapPid) -> From fcf046eec16356bdb2398081a43254f4db159259 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 09:39:23 +0800 Subject: [PATCH 20/72] chore(appup): supply the appup instructions --- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m.appup.src diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src new file mode 100644 index 000000000..fb22c843d --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. From e0068f6ebea0bee1c6be2d16bc37823dd2e0e0a2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:53:07 +0800 Subject: [PATCH 21/72] chore(prom): update grafana templates --- .../emqx_prometheus/grafana_template/EMQ.json | 1048 +++++++++++++- .../grafana_template/EMQ_Dashboard.json | 111 +- .../grafana_template/ErlangVM.json | 1217 +++++++++++++---- 3 files changed, 2050 insertions(+), 326 deletions(-) diff --git a/apps/emqx_prometheus/grafana_template/EMQ.json b/apps/emqx_prometheus/grafana_template/EMQ.json index 41dbb0be1..137e3a5a4 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ.json +++ b/apps/emqx_prometheus/grafana_template/EMQ.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1589443996970, + "iteration": 1620639954651, "links": [], "panels": [ { @@ -26,6 +26,12 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -54,9 +60,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -66,13 +73,22 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_connections_count{instance=\"$host\"}", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Clients", + "legendFormat": "Current", "refId": "A", "step": 60 + }, + { + "exemplar": true, + "expr": "emqx_connections_max{instance=\"$host\"}", + "hide": false, + "interval": "1m", + "legendFormat": "Max", + "refId": "B" } ], "thresholds": [], @@ -122,7 +138,11 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -132,6 +152,228 @@ "y": 0 }, "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_sessions_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_sessions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Session", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_suboptions_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Subopt-Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_suboptions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Subopt-Max", + "refId": "B" + }, + { + "exemplar": true, + "expr": "emqx_subscribers_count{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Suber-Current", + "refId": "C" + }, + { + "exemplar": true, + "expr": "emqx_subscribers_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Suber-Max", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Subscribers & Suboptions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, "id": 7, "legend": { "avg": false, @@ -147,9 +389,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -159,12 +402,38 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_subscriptions_count{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "Subscriptions", + "legendFormat": "Normal-Current", "refId": "A", "step": 60 + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Normal-Max", + "refId": "B" + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_shared_count{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Shared-Current", + "refId": "C" + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_shared_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Shared-Max", + "refId": "D" } ], "thresholds": [], @@ -215,13 +484,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 7 + "y": 14 }, "hiddenSeries": false, "id": 12, @@ -239,9 +514,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -251,11 +527,21 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_routes_count{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "Topics", + "legendFormat": "Current", "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_routes_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" } ], "thresholds": [], @@ -305,14 +591,123 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 7 + "y": 14 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_retained_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_retained_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Retained Messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 20 }, "hiddenSeries": false, "id": 13, @@ -330,9 +725,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -342,6 +738,7 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -350,6 +747,7 @@ "refId": "A" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -358,6 +756,7 @@ "refId": "B" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_retained{instance=\"$host\"}[$__interval]) or irate(emqx_messages_retained{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -366,19 +765,36 @@ "refId": "C" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "Dropped", "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_delayed{instance=\"$host\"}[$__interval]) or irate(emqx_messages_delayed{instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delayed", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_forward{instance=\"$host\"}[$__interval]) or irate(emqx_messages_forward{instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Forward", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Messages", + "title": "Messages Overview", "tooltip": { "shared": true, "sort": 0, @@ -422,13 +838,433 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Message.Dropped", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Message.Dropped.Expired", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Message.Dropped.no_subscribers", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_no_local{instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_no_local {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.no_local", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.too_large", + "refId": "F" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.qos0_msg", + "refId": "G" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.queue_full", + "refId": "H" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.expired", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages/Delivery Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, "w": 12, "x": 0, - "y": 14 + "y": 26 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Received", + "refId": "A" + }, + { + "exemplar": false, + "expr": "sum(rate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS0", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS1", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Sent", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS0", + "refId": "F" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS1", + "refId": "G" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS2", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 32 }, "hiddenSeries": false, "id": 9, @@ -447,9 +1283,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -524,13 +1361,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 14 + "y": 32 }, "hiddenSeries": false, "id": 2, @@ -554,9 +1397,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -566,7 +1410,8 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_connect{instance=\"$host\"}[$__interval]) or irate(emqx_packets_connect{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_connect{instance=\"$host\"}[$__interval])", "format": "time_series", "hide": false, "interval": "$interval", @@ -576,7 +1421,8 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -585,13 +1431,47 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Disconnect sent", "refId": "C", "step": 60 + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_connack_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "connack_error", + "refId": "D" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_connack_auth_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "connack_auth_error", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_auth_received {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Auth", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_auth_sent {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Auth sent", + "refId": "G" } ], "thresholds": [], @@ -642,13 +1522,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 21 + "y": 38 }, "hiddenSeries": false, "hideTimeOverride": false, @@ -671,9 +1557,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -684,8 +1571,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval])", "format": "time_series", + "instant": false, "interval": "$interval", "intervalFactor": 1, "legendFormat": "Subscribe received", @@ -694,7 +1583,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -703,7 +1593,8 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -712,13 +1603,38 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_unsuback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_unsuback_send{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_unsuback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "Unsuback sent", "refId": "D", "step": 60 + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_auth_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "sub_auth_error", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "sub_error", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_unsubscribe_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "unsub_error", + "refId": "G" } ], "thresholds": [], @@ -769,13 +1685,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 21 + "y": 38 }, "hiddenSeries": false, "id": 10, @@ -794,9 +1716,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -873,13 +1796,19 @@ "dashes": false, "datasource": "Prometheus", "decimals": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 28 + "y": 44 }, "hiddenSeries": false, "hideTimeOverride": false, @@ -900,9 +1829,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -912,7 +1842,8 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_publish_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_publish_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_publish_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -921,7 +1852,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -930,7 +1862,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_puback_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_puback_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_puback_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -940,7 +1873,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -949,7 +1883,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubrel_received{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubrel_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubrel_received{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -958,7 +1893,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubrel_sent{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubrel_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubrel_sent{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -967,7 +1903,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubcomp_received{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubcomp_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubcomp_received{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -976,7 +1913,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -1029,7 +1967,7 @@ } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -1043,6 +1981,8 @@ "text": "auto", "value": "$__auto_interval_interval" }, + "description": null, + "error": null, "hide": 0, "label": "Interval", "name": "interval", @@ -1096,19 +2036,24 @@ { "allValue": null, "current": { - "text": "emqx~127.0.0.1", - "value": "emqx~127.0.0.1" + "selected": false, + "text": "128.160.171.92:8081", + "value": "128.160.171.92:8081" }, "datasource": "Prometheus", "definition": "label_values(emqx_connections_count, instance)", + "description": null, + "error": null, "hide": 0, "includeAll": false, - "index": -1, "label": "Host", "multi": false, "name": "host", "options": [], - "query": "label_values(emqx_connections_count, instance)", + "query": { + "query": "label_values(emqx_connections_count, instance)", + "refId": "Prometheus-host-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1153,8 +2098,5 @@ "timezone": "browser", "title": "EMQ", "uid": "tjRlQw6Zk", - "variables": { - "list": [] - }, - "version": 6 + "version": 29 } \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json index 661f4d739..0b0e2036b 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json +++ b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 1, + "id": 3, "links": [], "panels": [ { @@ -25,10 +25,16 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 10, "w": 24, "x": 0, "y": 0 @@ -53,9 +59,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -65,23 +72,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(emqx_client_connected{instance=~\".*\",job=\"emqx\"})", + "exemplar": true, + "expr": "sum(emqx_connections_count{instance=~\".*\",job=\"emqx\"})", + "hide": false, + "interval": "", + "legendFormat": "Clients", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_client_connected{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "instant": false, "interval": "1m", "intervalFactor": 1, - "legendFormat": "connected", - "refId": "A", + "legendFormat": "connected rate", + "refId": "B", "step": 60 }, { - "expr": "sum(emqx_client_disconnected{instance=~\".*\",job=\"emqx\"})", + "exemplar": true, + "expr": "sum(rate(emqx_client_disconnected{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "instant": false, "interval": "1m", "intervalFactor": 1, - "legendFormat": "disconnected", - "refId": "B", + "legendFormat": "disconnected rate", + "refId": "C", "step": 60 } ], @@ -136,13 +153,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 24, "x": 0, - "y": 7 + "y": 10 }, "hiddenSeries": false, "id": 11, @@ -161,9 +184,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -173,20 +197,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_messages_received{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_messages_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Messages received", + "legendFormat": "received", "refId": "A", "step": 120 }, { - "expr": "sum(rate(emqx_messages_sent{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_messages_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Messages sent", + "legendFormat": "sent", "refId": "B", "step": 120 } @@ -240,13 +266,19 @@ "dashes": false, "datasource": "Prometheus", "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 14 + "y": 18 }, "hiddenSeries": false, "id": 13, @@ -266,9 +298,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -337,13 +370,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 21 + "y": 25 }, "hiddenSeries": false, "id": 8, @@ -362,9 +401,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -374,7 +414,8 @@ "steppedLine": false, "targets": [ { - "expr": "emqx_subscribers_count{instance=~\".*\",job=\"emqx\"}", + "exemplar": true, + "expr": "emqx_subscriptions_count{instance=~\".*\",job=\"emqx\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -443,13 +484,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 28 + "y": 32 }, "hiddenSeries": false, "id": 9, @@ -468,9 +515,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -480,18 +528,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_bytes_received{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_bytes_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "", "intervalFactor": 1, "legendFormat": "Bytes received", "refId": "A", "step": 60 }, { - "expr": "sum(rate(emqx_bytes_sent{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_bytes_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "", "intervalFactor": 1, "legendFormat": "Bytes sent", "refId": "B", @@ -541,7 +591,7 @@ } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -579,8 +629,5 @@ "timezone": "browser", "title": "EMQ Dashboard", "uid": "5sreUw6Wz", - "variables": { - "list": [] - }, - "version": 6 + "version": 11 } \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/ErlangVM.json b/apps/emqx_prometheus/grafana_template/ErlangVM.json index 1e9321f84..556d815b0 100644 --- a/apps/emqx_prometheus/grafana_template/ErlangVM.json +++ b/apps/emqx_prometheus/grafana_template/ErlangVM.json @@ -16,8 +16,8 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 3, - "iteration": 1589444007168, + "id": 4, + "iteration": 1620639927619, "links": [], "panels": [ { @@ -25,17 +25,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", - "fill": 0, + "datasource": null, + "description": "The CPU usage for the hosted machine", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, "y": 0 }, "hiddenSeries": false, - "id": 1, + "id": 14, "legend": { "avg": false, "current": false, @@ -47,13 +52,13 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pointradius": 5, + "pluginVersion": "7.5.5", + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -62,27 +67,26 @@ "steppedLine": false, "targets": [ { - "expr": "erlang_vm_process_count{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Process count", - "refId": "A", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_cpu_use {instance=\"$host\"}", + "interval": "", + "legendFormat": "used", + "refId": "A" }, { - "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Run queue length", - "refId": "B", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_cpu_idle {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Process", + "title": "CPU", "tooltip": { "shared": true, "sort": 0, @@ -102,7 +106,205 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "The memory usage for the hosted machine", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_vm_total_memory{instance=\"$host\"}", + "interval": "", + "legendFormat": "total", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_vm_used_memory{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "used", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "The total amount of memory currently allocated.\nThis is the same as the sum of the memory size for processes and system.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_memory_bytes_total {instance=\"$host\"}", + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VM Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, "show": true }, { @@ -125,203 +327,20 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "The total amount of memory currently allocated for the Erlang processes.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "erlang_vm_threads{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Threads", - "refId": "A", - "step": 60 - }, - { - "expr": "erlang_vm_thread_pool_size{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Thread pool size", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "erlang_vm_memory_system_bytes_total{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{usage}}", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory system", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 + "y": 8 }, "hiddenSeries": false, "id": 6, @@ -339,9 +358,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -405,16 +425,23 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "The total amount of memory currently allocated for the emulator that is not directly related to any Erlang process.\nMemory presented as processes is not included in this memory.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 15 }, "hiddenSeries": false, - "id": 4, + "id": 2, "legend": { "avg": false, "current": false, @@ -429,9 +456,568 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$host\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{usage}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory system", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_logical_processors {instance=\"$host\"}", + "interval": "", + "legendFormat": "logical-processors", + "refId": "A" + }, + { + "exemplar": true, + "expr": "erlang_vm_logical_processors_available {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "logical-processors-avai", + "refId": "B" + }, + { + "exemplar": true, + "expr": "erlang_vm_logical_processors_online {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "logical-processors-online", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_vm_schedulers {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "schedulers", + "refId": "D" + }, + { + "exemplar": true, + "expr": "erlang_vm_schedulers_online {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "schedulers-online", + "refId": "E" + }, + { + "exemplar": true, + "expr": "erlang_vm_threads {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_threads", + "refId": "F" + }, + { + "exemplar": true, + "expr": "erlang_vm_thread_pool_size {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_thread_pool_size", + "refId": "G" + }, + { + "exemplar": true, + "expr": "emqx_vm_run_queue {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_run_queue", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Processor & Scheduler & Thread", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_ets_limit {instance=\"$host\"}", + "interval": "", + "legendFormat": "ETS Max", + "refId": "A" + }, + { + "exemplar": true, + "expr": "erlang_vm_ets_tables {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "ETS Count", + "refId": "B" + }, + { + "exemplar": true, + "expr": "erlang_vm_dets_tables {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "DETS Count", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_vm_process_limit {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Process Max", + "refId": "D" + }, + { + "exemplar": true, + "expr": "erlang_vm_process_count {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Process Count", + "refId": "E" + }, + { + "exemplar": true, + "expr": "erlang_vm_port_limit {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Port Max", + "refId": "F" + }, + { + "exemplar": true, + "expr": "erlang_vm_port_count {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Port Count", + "refId": "G" + }, + { + "exemplar": true, + "expr": "erlang_vm_smp_support {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "smp_support", + "refId": "H" + }, + { + "exemplar": true, + "expr": "erlang_vm_time_correction {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "time_correction", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_context_switches{instance=\"$host\"}[$__interval])", + "interval": "", + "legendFormat": "context_switches", + "refId": "A" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_garbage_collection_number_of_gcs {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "garbage_collection_number_of_gcs", + "refId": "B" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_garbage_collection_bytes_reclaimed {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "garbage_collection_bytes_reclaimed", + "refId": "D" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_bytes_received_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "bytes_received_total", + "refId": "C" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_bytes_output_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "bytes_output_total", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_reductions_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "reductions", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_runtime_milliseconds {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "runtime_milliseconds", + "refId": "G" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_wallclock_time_milliseconds {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "wallclock_time_milliseconds", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VM Statistics", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -441,35 +1027,35 @@ "steppedLine": false, "targets": [ { - "expr": "erlang_vm_ets_tables{instance=\"$host\"}", + "expr": "erlang_vm_process_count{instance=\"$host\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "ETS tables", + "legendFormat": "Process count", "refId": "A", "step": 60 }, { - "expr": "erlang_vm_ets_limit{instance=\"$host\"}", + "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$host\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "ETS limit", + "legendFormat": "Run queue length", "refId": "B", "step": 60 }, { - "expr": "erlang_vm_dets_tables{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "DETS tables", - "refId": "C", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_process_messages_in_queues {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "messages_in_queue", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "ETS", + "title": "Process", "tooltip": { "shared": true, "sort": 0, @@ -512,13 +1098,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 29 }, "hiddenSeries": false, "id": 5, @@ -536,9 +1128,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -556,12 +1149,30 @@ "step": 30 }, { + "exemplar": true, "expr": "erlang_mnesia_held_locks{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 2, "legendFormat": "Held locks", "refId": "B", "step": 30 + }, + { + "exemplar": true, + "expr": "erlang_mnesia_transaction_coordinators{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Transaction coordinators", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_mnesia_transaction_participants {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Transaction participants", + "refId": "D" } ], "thresholds": [], @@ -605,10 +1216,129 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]))", + "interval": "", + "legendFormat": "Committed", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Failed", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Restart", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Logged", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mnesia transaction", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -618,10 +1348,12 @@ "auto_count": 200, "auto_min": "1s", "current": { - "selected": false, + "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, + "description": null, + "error": null, "hide": 0, "label": "Interval", "name": "interval", @@ -668,6 +1400,7 @@ } ], "query": "1s,5s,1m,5m,1h,6h,1d", + "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" @@ -675,19 +1408,24 @@ { "allValue": null, "current": { - "text": "emqx~127.0.0.1", - "value": "emqx~127.0.0.1" + "selected": false, + "text": "128.160.171.92:8081", + "value": "128.160.171.92:8081" }, "datasource": "Prometheus", "definition": "", + "description": null, + "error": null, "hide": 0, "includeAll": false, - "index": -1, "label": "Host", "multi": false, "name": "host", "options": [], - "query": "label_values(erlang_vm_process_count, instance)", + "query": { + "query": "label_values(erlang_vm_process_count, instance)", + "refId": "Prometheus-host-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -732,8 +1470,5 @@ "timezone": "browser", "title": "ErlangVM", "uid": "stprQQ6Zk", - "variables": { - "list": [] - }, - "version": 1 + "version": 13 } \ No newline at end of file From 85937e00073617628152232228c474acadeae480 Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 12 May 2021 22:09:57 +0800 Subject: [PATCH 22/72] fix(import): fix import bridge mqtt test cases --- .../test/emqx_bridge_mqtt_data_export_import_SUITE.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index 7cf7d19cf..aca60a687 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -177,7 +177,6 @@ handle_config(Config, ee435, Id) -> remove_resources() -> timer:sleep(500), lists:foreach(fun(#resource{id = Id}) -> - emqx_rule_registry:remove_resource(Id), - emqx_rule_registry:remove_resource_params(Id) + emqx_rule_engine:delete_resource(Id) end, emqx_rule_registry:get_resources()), timer:sleep(500). \ No newline at end of file From 6ba1a525e03626082e53f34c3cee7d6e4cab07ef Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 13 May 2021 09:27:18 +0800 Subject: [PATCH 23/72] fix(lwm2m): fix check emqx-lwm2m version --- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index a576a4149..19fabc526 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, From 9c244679dcbc316ba8931fa2b197e7bd3af507d8 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 13 May 2021 13:51:17 +0800 Subject: [PATCH 24/72] chore(CI): relup test support any commit --- .ci/fvt_tests/relup.lux | 23 ++++++----------------- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/run_fvt_tests.yaml | 14 ++++++++------ 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index c0c2b7592..b75e0fa94 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -21,20 +21,15 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o emqx-ubuntu20.04-$old_vsn-x86_64.zip + !unzip -q -o emqx-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip ?SH-PROMPT !cd emqx !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X Broker $old_vsn is started successfully! - - !./bin/emqx_ctl status - """? - Node 'emqx@127.0.0.1' is started - emqx $old_vsn is running - """ + ?EMQ X .* is started successfully! + ?SH-PROMPT [shell emqx2] !cd $PACKAGE_PATH @@ -45,13 +40,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X Broker $old_vsn is started successfully! - - !./bin/emqx_ctl status - """? - Node 'emqx2@127.0.0.1' is started - emqx $old_vsn is running - """ + ?EMQ X (.*) is started successfully! ?SH-PROMPT !./bin/emqx_ctl cluster join emqx@127.0.0.1 @@ -86,7 +75,7 @@ ???sent [shell emqx] - !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" @@ -101,7 +90,7 @@ ?SH-PROMPT [shell emqx2] - !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fde6795b1..70419bcc1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -256,9 +256,9 @@ jobs: if [ $PROFILE = "emqx" ];then broker="emqx-ce"; else broker="$PROFILE"; fi if [ $PROFILE = "emqx-ee" ];then edition='enterprise'; else edition='opensource'; fi - vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" + vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx-ee" ]; then + if [ $PROFILE = "emqx-ee" ]; then old_vsns=($(git tag -l "e$pre_vsn.[0-9]" | sed "s/e$vsn//")) else old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//")) @@ -268,8 +268,8 @@ jobs: cd _upgrade_base for tag in ${old_vsns[@]};do if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 fi done diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8f65d022e..eea7acde5 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -214,24 +214,26 @@ jobs: broker="emqx-ee" edition='enterprise' fi + echo "BROKER=$broker" >> $GITHUB_ENV - vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" + vsn="$(./pkg-vsn.sh)" echo "VSN=$vsn" >> $GITHUB_ENV pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx" ]; then + if [ $PROFILE = "emqx" ]; then old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" else - old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" + old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" fi echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV - name: download emqx run: | set -e -x -u - cd emqx + mkdir -p emqx/_upgrade_base + cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-x86_64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip done - name: build emqx run: make -C emqx ${PROFILE}-zip @@ -251,7 +253,7 @@ jobs: if [ -n "$OLD_VSNS" ]; then mkdir -p packages cp emqx/_packages/emqx/*.zip packages - cp emqx/*.zip packages + cp emqx/_upgrade_base/*.zip packages lux -v \ --timeout 600000 \ --var PACKAGE_PATH=$(pwd)/packages \ From 53291ac3ff846cde139daf9c506c28d425ac7481 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 06:58:08 +0200 Subject: [PATCH 25/72] fix(nodetool): add path from RELEASES file --- bin/nodetool | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/bin/nodetool b/bin/nodetool index 5ab5ec69a..431121148 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -296,11 +296,30 @@ add_libs_dir() -> os:getenv("REL_VSN"), "emqx.rel" ]), - {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} = file:consult(RelFile), - lists:foreach( - fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); - ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) - end, Libs). + case file:consult(RelFile) of + {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} -> + lists:foreach( + fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); + ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) + end, Libs); + {error, enoent} -> + %% rel file is deleted by release handler + add_libs_dir2(RootDir) + end. + +add_libs_dir2(RootDir) -> + RelFile = filename:join([RootDir, "releases", "RELEASES"]), + case file:consult(RelFile) of + {ok, [[Release]]} -> + {release, _Name, _AppVsn, _ErtsVsn, Libs, _State} = Release, + lists:foreach( + fun({Name, Vsn, _}) -> + add_lib_dir(RootDir, Name, Vsn) + end, Libs); + {error, Reason} -> + %% rel file was been deleted by release handler + error({failed_to_read_RELEASES_file, RelFile, Reason}) + end. add_lib_dir(RootDir, Name, Vsn) -> LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), From c929306cb9a9763862e5a136d9ef7127892686e7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 May 2021 19:02:21 +0800 Subject: [PATCH 26/72] chore(metrics): update ets table in the metrics proc --- src/emqx_metrics.erl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 3b96bfbad..a37e952e4 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -201,16 +201,8 @@ stop() -> gen_server:stop(?SERVER). %% BACKW: v4.3.0 upgrade_retained_delayed_counter_type() -> - case ets:info(?TAB, name) of - ?TAB -> - [M1] = ets:lookup(?TAB, 'messages.retained'), - [M2] = ets:lookup(?TAB, 'messages.delayed'), - true = ets:insert(?TAB, M1#metric{type = counter}), - true = ets:insert(?TAB, M2#metric{type = counter}), - ok; - _ -> - ok - end. + Ks = ['messages.retained', 'messages.delayed'], + gen_server:call(?SERVER, {set_type_to_counter, Ks}). %%-------------------------------------------------------------------- %% Metrics API @@ -467,6 +459,13 @@ handle_call({create, Type, Name}, _From, State = #state{next_idx = NextIdx}) -> {reply, {ok, NextIdx}, State#state{next_idx = NextIdx + 1}} end; +handle_call({set_type_to_counter, Keys}, _From, State) -> + lists:foreach( + fun(K) -> + ets:update_element(?TAB, K, {#metric.type, counter}) + end, Keys), + {reply, ok, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. From a1162249b570864a46379c411c1e44891b1bdffb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 17:11:59 +0200 Subject: [PATCH 27/72] fix(nodetool): find release in releases --- bin/nodetool | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/bin/nodetool b/bin/nodetool index 431121148..0e89ac278 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -292,25 +292,11 @@ join([H|T], Sep) -> add_libs_dir() -> [_ | _] = RootDir = os:getenv("RUNNER_ROOT_DIR"), - RelFile = filename:join([RootDir, "releases", - os:getenv("REL_VSN"), - "emqx.rel" - ]), - case file:consult(RelFile) of - {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} -> - lists:foreach( - fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); - ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) - end, Libs); - {error, enoent} -> - %% rel file is deleted by release handler - add_libs_dir2(RootDir) - end. - -add_libs_dir2(RootDir) -> + CurrentVsn = os:getenv("REL_VSN"), RelFile = filename:join([RootDir, "releases", "RELEASES"]), case file:consult(RelFile) of - {ok, [[Release]]} -> + {ok, [Releases]} -> + Release = lists:keyfind(CurrentVsn, 3, Releases), {release, _Name, _AppVsn, _ErtsVsn, Libs, _State} = Release, lists:foreach( fun({Name, Vsn, _}) -> From 1e3f7319ac8c32b0f71f7609d56af61912f02a5b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 17:16:54 +0200 Subject: [PATCH 28/72] fix(emqx_metrics): infinity gen_server call in upgrade --- src/emqx_metrics.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index a37e952e4..c3ce14d83 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -202,7 +202,7 @@ stop() -> gen_server:stop(?SERVER). %% BACKW: v4.3.0 upgrade_retained_delayed_counter_type() -> Ks = ['messages.retained', 'messages.delayed'], - gen_server:call(?SERVER, {set_type_to_counter, Ks}). + gen_server:call(?SERVER, {set_type_to_counter, Ks}, infinity). %%-------------------------------------------------------------------- %% Metrics API From de43da881a26346ae4c091408ff6a04fedd89bc6 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 22:52:04 +0200 Subject: [PATCH 29/72] fix(emqx_frame): poor large frame concatenation performance piror to this change, binary concatenation eats most of the CPU --- src/emqx_connection.erl | 2 +- src/emqx_frame.erl | 73 ++++++++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 69ffddf06..89bb06e12 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -475,7 +475,7 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, E : C : S -> ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) end, - ?tp(debug, terminate, #{}), + ?tp(info, terminate, #{reason => Reason}), maybe_raise_excption(Reason). %% close socket, discard new state, always return ok. diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 7e5aae788..c4a2f6ac0 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -40,6 +40,8 @@ , serialize_opts/0 ]). +-define(Q(BYTES, Q), {BYTES, Q}). + -type(options() :: #{strict_mode => boolean(), max_size => 1..?MAX_PACKET_SIZE, version => emqx_types:version() @@ -50,12 +52,12 @@ -type(parse_result() :: {more, parse_state()} | {ok, emqx_types:packet(), binary(), parse_state()}). --type(cont_state() :: {Stage :: len | body, - State :: #{hdr := #mqtt_packet_header{}, - len := {pos_integer(), non_neg_integer()} | non_neg_integer(), - rest => binary() - } - }). +-type(cont_state() :: + {Stage :: len | body, + State :: #{hdr := #mqtt_packet_header{}, + len := {pos_integer(), non_neg_integer()} | non_neg_integer(), + rest => binary() | ?Q(non_neg_integer(), queue:queue(binary())) + }}). -type(serialize_opts() :: options()). @@ -117,9 +119,19 @@ parse(Bin, {{len, #{hdr := Header, parse_remaining_len(Bin, Header, Multiplier, Length, Options); parse(Bin, {{body, #{hdr := Header, len := Length, - rest := Rest} + rest := Body} }, Options}) when is_binary(Bin) -> - parse_frame(<>, Header, Length, Options). + BodyBytes = body_bytes(Body), + {NewBodyPart, Tail} = split(BodyBytes + size(Bin) - Length, Bin), + NewBody = append_body(Body, NewBodyPart), + parse_frame(NewBody, Tail, Header, Length, Options). + +%% split given binary with the first N bytes +split(N, Bin) when N =< 0 -> + {Bin, <<>>}; +split(N, Bin) when N =< size(Bin) -> + <> = Bin, + {H, T}. parse_remaining_len(<<>>, Header, Options) -> {more, {{len, #{hdr => Header, len => {1, 0}}}, Options}}; @@ -132,7 +144,8 @@ parse_remaining_len(_Bin, _Header, _Multiplier, Length, #{max_size := MaxSize}) parse_remaining_len(<<>>, Header, Multiplier, Length, Options) -> {more, {{len, #{hdr => Header, len => {Multiplier, Length}}}, Options}}; %% Match DISCONNECT without payload -parse_remaining_len(<<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?DISCONNECT}, 1, 0, Options) -> +parse_remaining_len(<<0:8, Rest/binary>>, + Header = #mqtt_packet_header{type = ?DISCONNECT}, 1, 0, Options) -> Packet = packet(Header, #mqtt_packet_disconnect{reason_code = ?RC_SUCCESS}), {ok, Packet, Rest, ?none(Options)}; %% Match PINGREQ. @@ -149,16 +162,35 @@ parse_remaining_len(<<1:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Opti parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Options = #{max_size := MaxSize}) -> FrameLen = Value + Len * Multiplier, - if - FrameLen > MaxSize -> error(frame_too_large); - true -> parse_frame(Rest, Header, FrameLen, Options) + case FrameLen > MaxSize of + true -> error(frame_too_large); + false -> parse_frame(Rest, Header, FrameLen, Options) end. -parse_frame(Bin, Header, 0, Options) -> - {ok, packet(Header), Bin, ?none(Options)}; -parse_frame(Bin, Header, Length, Options) -> - case Bin of - <> -> +body_bytes(B) when is_binary(B) -> size(B); +body_bytes(?Q(Bytes, _)) -> Bytes. + +append_body(H, T) when is_binary(H) andalso size(H) < 1024 -> + <>; +append_body(H, T) when is_binary(H) -> + Bytes = size(H) + size(T), + ?Q(Bytes, queue:from_list([H, T])); +append_body(?Q(Bytes, Q), T) -> + ?Q(Bytes + iolist_size(T), queue:in(T, Q)). + +flatten_body(Body, Tail) when is_binary(Body) -> <>; +flatten_body(?Q(_, Q), Tail) -> iolist_to_binary([queue:to_list(Q), Tail]). + +parse_frame(Body, Header, Length, Options) -> + %% already appended + parse_frame(Body, _SplitTail = <<>>, Header, Length, Options). + +parse_frame(Body, Tail, Header, 0, Options) -> + {ok, packet(Header), flatten_body(Body, Tail), ?none(Options)}; +parse_frame(Body, Tail, Header, Length, Options) -> + case body_bytes(Body) >= Length of + true -> + <> = flatten_body(Body, Tail), case parse_packet(Header, FrameBin, Options) of {Variable, Payload} -> {ok, packet(Header, Variable, Payload), Rest, ?none(Options)}; @@ -167,8 +199,11 @@ parse_frame(Bin, Header, Length, Options) -> Variable -> {ok, packet(Header, Variable), Rest, ?none(Options)} end; - TooShortBin -> - {more, {{body, #{hdr => Header, len => Length, rest => TooShortBin}}, Options}} + false -> + {more, {{body, #{hdr => Header, + len => Length, + rest => append_body(Body, Tail) + }}, Options}} end. -compile({inline, [packet/1, packet/2, packet/3]}). From c64b00720fb6c79cefe84cbf2a54098503241a18 Mon Sep 17 00:00:00 2001 From: Rory-Z Date: Fri, 14 May 2021 05:42:59 +0000 Subject: [PATCH 30/72] chore(CI): update relup test script --- .ci/fvt_tests/relup.lux | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index b75e0fa94..85b47bb60 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -111,13 +111,6 @@ ???{"data":600,"code":0} ?SH-PROMPT -[shell http_server] - !http_server:stop(). - ?ok - ?> - !halt(3). - ?SH-PROMPT: - [shell emqx2] !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100 -error @@ -142,6 +135,13 @@ !rm -rf $PACKAGE_PATH/emqx ?SH-PROMPT: +[shell http_server] + !http_server:stop(). + ?ok + ?> + !halt(3). + ?SH-PROMPT: + [endloop] [cleanup] From 7f2bb9b4ee5d635f341c74affb9541ff18208b09 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 14 May 2021 09:45:46 +0200 Subject: [PATCH 31/72] fix(emqx): Add new shared_subscription_strategy atoms to the schema Fixes: #4808 --- priv/emqx.schema | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index aee2fddc6..e5c58f216 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2280,7 +2280,9 @@ end}. [random, %% randomly pick a subscriber round_robin, %% round robin alive subscribers one message after another sticky, %% pick a random subscriber and stick to it - hash %% hash client ID to a group member + hash, %% hash client ID to a group member + hash_clientid, + hash_topic ]}} ]}. From d0971ceb53f57f2649a568853db59d138d0fb18c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 14 May 2021 10:42:11 +0200 Subject: [PATCH 32/72] fix: add emqx appup --- src/emqx.appup.src | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b10c14a9c..d162cc15d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,6 +3,8 @@ [ {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + {load_module, emqx_connection, brutal_purge, soft_purge, []}, + {load_module, emqx_frame, brutal_purge, soft_purge, []}, {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, @@ -11,6 +13,8 @@ [ {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + {load_module, emqx_connection, brutal_purge, soft_purge, []}, + {load_module, emqx_frame, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []} From 30990edbd474dcab81e67f195253d9c5eb83d43c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 12 May 2021 20:32:16 +0200 Subject: [PATCH 33/72] fix(emqx_broker_bench): test real match performance --- src/emqx_broker_bench.erl | 148 ++++++++++++++++++++++++++++---------- 1 file changed, 110 insertions(+), 38 deletions(-) diff --git a/src/emqx_broker_bench.erl b/src/emqx_broker_bench.erl index 45ef0eab6..725a65898 100644 --- a/src/emqx_broker_bench.erl +++ b/src/emqx_broker_bench.erl @@ -18,32 +18,83 @@ -ifdef(EMQX_BENCHMARK). --export([start/1, run1/0, run1/2]). +-export([run/1, run1/0, run1/4]). -run1() -> run1(4, 1000). +-define(T(Expr), timer:tc(fun() -> Expr end)). -run1(Factor, Limit) -> - start(#{factor => Factor, - limit => Limit, - sub_ptn => <<"device/{{id}}/+/{{num}}/#">>, - pub_ptn => <<"device/{{id}}/xays/{{num}}/foo/bar/baz">>}). +run1() -> run1(80, 1000, 80, 10000). + +run1(Subs, SubOps, Pubs, PubOps) -> + run(#{subscribers => Subs, + publishers => Pubs, + sub_ops => SubOps, + pub_ops => PubOps, + sub_ptn => <<"device/{{id}}/+/{{num}}/#">>, + pub_ptn => <<"device/{{id}}/foo/{{num}}/bar">> + }). %% setting fields: -%% - factor: spawn broker-pool-size * factor number of callers -%% - limit: limit the total number of topics for each caller +%% - subscribers: spawn this number of subscriber workers +%% - publishers: spawn this number of publisher workers +%% - sub_ops: the number of subscribes (route insert) each subscriber runs +%% - pub_ops: the number of publish (route lookups) each publisher runs %% - sub_ptn: subscribe topic pattern like a/+/b/+/c/# %% or a/+/{{id}}/{{num}}/# to generate pattern with {{id}} %% replaced by worker id and {{num}} replaced by topic number. %% - pub_ptn: topic pattern used to benchmark publish (match) performance %% e.g. a/x/{{id}}/{{num}}/foo/bar -start(#{factor := Factor} = Settings) -> - BrokerPoolSize = emqx_vm:schedulers() * 2, - Pids = start_callers(BrokerPoolSize * Factor, Settings), - R = collect_results(Pids, #{subscribe => 0, match => 0}), +run(#{subscribers := Subs, + publishers := Pubs, + sub_ops := SubOps, + pub_ops := PubOps + } = Settings) -> + SubsPids = start_callers(Subs, fun start_subscriber/1, Settings), + PubsPids = start_callers(Pubs, fun start_publisher/1, Settings), + _ = collect_results(SubsPids, subscriber_ready), + io:format(user, "subscribe ...~n", []), + {T1, SubsTime} = + ?T(begin + lists:foreach(fun(Pid) -> Pid ! start_subscribe end, SubsPids), + collect_results(SubsPids, subscribe_time) + end), + io:format(user, "InsertTotalTime: ~s~n", [ns(T1)]), + io:format(user, "InsertTimeAverage: ~s~n", [ns(SubsTime / Subs)]), + io:format(user, "InsertRps: ~p~n", [rps(Subs * SubOps, T1)]), + + io:format(user, "lookup ...~n", []), + {T2, PubsTime} = + ?T(begin + lists:foreach(fun(Pid) -> Pid ! start_lookup end, PubsPids), + collect_results(PubsPids, lookup_time) + end), + io:format(user, "LookupTotalTime: ~s~n", [ns(T2)]), + io:format(user, "LookupTimeAverage: ~s~n", [ns(PubsTime / Pubs)]), + io:format(user, "LookupRps: ~p~n", [rps(Pubs * PubOps, T2)]), + io:format(user, "mnesia table(s) RAM: ~p~n", [ram_bytes()]), - io:format(user, "~p~n", [erlang:memory()]), - io:format(user, "~p~n", [R]), - lists:foreach(fun(Pid) -> Pid ! stop end, Pids). + io:format(user, "erlang memory: ~p~n", [erlang:memory()]), + + io:format(user, "unsubscribe ...~n", []), + {T3, ok} = + ?T(begin + lists:foreach(fun(Pid) -> Pid ! stop end, SubsPids), + wait_until_empty() + end), + io:format(user, "TimeToUnsubscribeAll: ~s~n", [ns(T3)]). + +wait_until_empty() -> + case emqx_trie:empty() of + true -> ok; + false -> + timer:sleep(5), + wait_until_empty() + end. + +rps(N, NanoSec) -> N * 1_000_000 / NanoSec. + +ns(T) when T > 1_000_000 -> io_lib:format("~p(s)", [T / 1_000_000]); +ns(T) when T > 1_000 -> io_lib:format("~p(ms)", [T / 1_000]); +ns(T) -> io_lib:format("~p(ns)", [T]). ram_bytes() -> Wordsize = erlang:system_info(wordsize), @@ -56,48 +107,69 @@ ram_bytes() -> 0 end. -start_callers(0, _) -> []; -start_callers(N, Settings) -> - [start_caller(Settings#{id => N}) | start_callers(N - 1, Settings)]. +start_callers(N, F, Settings) -> + start_callers(N, F, Settings, []). -collect_results([], R) -> R; -collect_results([Pid | Pids], Acc = #{subscribe := Sr, match := Mr}) -> +start_callers(0, _F, _Settings, Acc) -> + lists:reverse(Acc); +start_callers(N, F, Settings, Acc) -> + start_callers(N - 1, F, Settings, [F(Settings#{id => N}) | Acc]). + +collect_results(Pids, Tag) -> + collect_results(Pids, Tag, 0). + +collect_results([], _Tag, R) -> R; +collect_results([Pid | Pids], Tag, R) -> receive - {Pid, #{subscribe := Srd, match := Mrd}} -> - collect_results(Pids, Acc#{subscribe := Sr + Srd, match := Mr + Mrd}) + {Pid, Tag, N} -> + collect_results(Pids, Tag, N + R) end. -%% ops per second -rps(T, N) -> round(N / (T / 1000000)). - -start_caller(#{id := Id, limit := N, sub_ptn := SubPtn, pub_ptn := PubPtn}) -> +start_subscriber(#{id := Id, sub_ops := N, sub_ptn := SubPtn}) -> Parent = self(), proc_lib:spawn_link( fun() -> SubTopics = make_topics(SubPtn, Id, N), - {Ts, _} = timer:tc(fun() -> subscribe(SubTopics) end), - PubTopics = make_topics(PubPtn, Id, N), - {Tm, _} = timer:tc(fun() -> match(PubTopics) end), - _ = erlang:send(Parent, {self(), #{subscribe => rps(Ts, N), match => rps(Tm, N)}}), + Parent ! {self(), subscriber_ready, 0}, + receive + start_subscribe -> + ok + end, + {Ts, _} = ?T(subscribe(SubTopics)), + _ = erlang:send(Parent, {self(), subscribe_time, Ts/ N}), + %% subscribers should not exit before publish test is done receive stop -> ok end end). -match([]) -> ok; -match([Topic | Topics]) -> - _ = emqx_router:lookup_routes(Topic), - match(Topics). +start_publisher(#{id := Id, pub_ops := N, pub_ptn := PubPtn, subscribers := Subs}) -> + Parent = self(), + proc_lib:spawn_link( + fun() -> + L = lists:seq(1, N), + [Topic] = make_topics(PubPtn, (Id rem Subs) + 1, 1), + receive + start_lookup -> + ok + end, + {Tm, ok} = ?T(lists:foreach(fun(_) -> match(Topic) end, L)), + _ = erlang:send(Parent, {self(), lookup_time, Tm / N}), + ok + end). + +match(Topic) -> + [_] = emqx_router:match_routes(Topic). subscribe([]) -> ok; subscribe([Topic | Rest]) -> ok = emqx_broker:subscribe(Topic), subscribe(Rest). -make_topics(SubPtn0, Id, Limit) -> - SubPtn = emqx_topic:words(SubPtn0), - F = fun(N) -> render(Id, N, SubPtn) end, +make_topics(Ptn0, Id, Limit) -> + Ptn = emqx_topic:words(Ptn0), + F = fun(N) -> render(Id, N, Ptn) end, lists:map(F, lists:seq(1, Limit)). render(ID, N, Ptn) -> From 3c03047c9f94c64185182a1ae91cfe0d0d2b38e9 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 15:15:22 +0200 Subject: [PATCH 34/72] fix(emqx_trie): performance issue when many levels --- src/emqx.appup.src | 2 ++ src/emqx_trie.erl | 50 ++++++++++++++++++++++++---------------- test/emqx_trie_SUITE.erl | 25 +++++++++++++++++++- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index d162cc15d..2cac1e315 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -5,6 +5,7 @@ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, + {load_module, emqx_trie, brutal_purge, soft_purge, []}, {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, @@ -15,6 +16,7 @@ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, + {load_module, emqx_trie, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []} diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 8ece333b0..901da2556 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -219,15 +219,22 @@ do_match(Words) -> do_match(Words, empty). do_match(Words, Prefix) -> - match(is_compact(), Words, Prefix, []). + case is_compact() of + true -> match_compact(Words, Prefix, []); + false -> match_no_compact(Words, Prefix, []) + end. -match(_IsCompact, [], Topic, Acc) -> +match_no_compact([], Topic, Acc) -> 'match_#'(Topic) ++ %% try match foo/bar/# lookup_topic(Topic) ++ %% try match foo/bar Acc; -match(IsCompact, [Word | Words], Prefix, Acc0) -> - case {has_prefix(Prefix), IsCompact} of - {false, false} -> +match_no_compact([Word | Words], Prefix, Acc0) -> + case has_prefix(Prefix) of + true -> + Acc1 = 'match_#'(Prefix) ++ Acc0, + Acc = match_no_compact(Words, join(Prefix, '+'), Acc1), + match_no_compact(Words, join(Prefix, Word), Acc); + false -> %% non-compact paths in database %% if there is no prefix matches the current topic prefix %% we can simpliy return from here @@ -240,21 +247,24 @@ match(IsCompact, [Word | Words], Prefix, Acc0) -> %% then at the second level, we lookup prefix a/x, %% no such prefix to be found, meaning there is no point %% searching for 'a/x/y', 'a/x/+' or 'a/x/#' - Acc0; - _ -> - %% compact paths in database - %% we have to enumerate all possible prefixes - %% e.g. a/+/b/# results with below entries in database - %% - a/+ - %% - a/+/b/# - %% when matching a/x/y, we need to enumerate - %% - a - %% - a/x - %% - a/x/y - %% *with '+', '#' replaced at each level - Acc1 = 'match_#'(Prefix) ++ Acc0, - Acc = match(IsCompact, Words, join(Prefix, '+'), Acc1), - match(IsCompact, Words, join(Prefix, Word), Acc) + Acc0 + end. + +match_compact([], Topic, Acc) -> + 'match_#'(Topic) ++ %% try match foo/bar/# + lookup_topic(Topic) ++ %% try match foo/bar + Acc; +match_compact([Word | Words], Prefix, Acc0) -> + Acc1 = 'match_#'(Prefix) ++ Acc0, + Acc = match_compact(Words, join(Prefix, Word), Acc1), + WildcardPrefix = join(Prefix, '+'), + %% go deeper to match current_prefix/+ only when: + %% 1. current word is the last + %% OR + %% 2. there is a prefix = 'current_prefix/+' + case Words =:= [] orelse has_prefix(WildcardPrefix) of + true -> match_compact(Words, WildcardPrefix, Acc); + false -> Acc end. 'match_#'(Prefix) -> diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 7516b58a0..23cd1009f 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -105,7 +105,10 @@ t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], trans(fun() -> [emqx_trie:insert(Topic) || Topic <- Topics] end), Matched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"a/b/c">>]), - ?assertEqual(4, length(Matched)), + case length(Matched) of + 4 -> ok; + _ -> error({unexpected, Matched}) + end, SysMatched = emqx_trie:match(<<"$SYS/a/b/c">>), ?assertEqual([<<"$SYS/#">>], SysMatched). @@ -114,6 +117,26 @@ t_match4(_) -> trans(fun() -> lists:foreach(fun emqx_trie:insert/1, Topics) end), ?assertEqual([<<"/#">>, <<"/+/a/b/c">>], lists:sort(emqx_trie:match(<<"/0/a/b/c">>))). +t_match5(_) -> + T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, + Topics = [<<"#">>, <>, <>], + trans(fun() -> lists:foreach(fun emqx_trie:insert/1, Topics) end), + ?assertEqual([<<"#">>, <>], lists:sort(emqx_trie:match(T))), + ?assertEqual([<<"#">>, <>, <>], + lists:sort(emqx_trie:match(<>))). + +t_match6(_) -> + T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, + W = <<"+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/#">>, + trans(fun() -> emqx_trie:insert(W) end), + ?assertEqual([W], emqx_trie:match(T)). + +t_match7(_) -> + T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>, + W = <<"a/+/c/+/e/+/g/+/i/+/k/+/m/+/o/+/q/+/s/+/u/+/w/+/y/+/#">>, + trans(fun() -> emqx_trie:insert(W) end), + ?assertEqual([W], emqx_trie:match(T)). + t_empty(_) -> ?assert(?TRIE:empty()), trans(fun ?TRIE:insert/1, [<<"topic/x/#">>]), From 55316b3ac36fd0d1a03b8a2839585520e90f3ba6 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 16:31:13 +0200 Subject: [PATCH 35/72] perf: micro optimisation: no lookup for non-wildcard in trie --- src/emqx_trie.erl | 31 ++++++++++++++++++------------- test/emqx_trie_SUITE.erl | 4 ++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 901da2556..7146feb74 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -194,6 +194,11 @@ delete_key(Key) -> ok end. +%% micro-optimization: no need to lookup when topic is not wildcard +%% because we only insert wildcards to emqx_trie +lookup_topic(_Topic, false) -> []; +lookup_topic(Topic, true) -> lookup_topic(Topic). + lookup_topic(Topic) when is_binary(Topic) -> case ets:lookup(?TRIE, ?TOPIC(Topic)) of [#?TRIE{count = C}] -> [Topic || C > 0]; @@ -220,20 +225,20 @@ do_match(Words) -> do_match(Words, Prefix) -> case is_compact() of - true -> match_compact(Words, Prefix, []); - false -> match_no_compact(Words, Prefix, []) + true -> match_compact(Words, Prefix, false, []); + false -> match_no_compact(Words, Prefix, false, []) end. -match_no_compact([], Topic, Acc) -> - 'match_#'(Topic) ++ %% try match foo/bar/# - lookup_topic(Topic) ++ %% try match foo/bar +match_no_compact([], Topic, IsWildcard, Acc) -> + 'match_#'(Topic) ++ %% try match foo/+/# or foo/bar/# + lookup_topic(Topic, IsWildcard) ++ %% e.g. foo/+ Acc; -match_no_compact([Word | Words], Prefix, Acc0) -> +match_no_compact([Word | Words], Prefix, IsWildcard, Acc0) -> case has_prefix(Prefix) of true -> Acc1 = 'match_#'(Prefix) ++ Acc0, - Acc = match_no_compact(Words, join(Prefix, '+'), Acc1), - match_no_compact(Words, join(Prefix, Word), Acc); + Acc = match_no_compact(Words, join(Prefix, '+'), true, Acc1), + match_no_compact(Words, join(Prefix, Word), IsWildcard, Acc); false -> %% non-compact paths in database %% if there is no prefix matches the current topic prefix @@ -250,20 +255,20 @@ match_no_compact([Word | Words], Prefix, Acc0) -> Acc0 end. -match_compact([], Topic, Acc) -> +match_compact([], Topic, IsWildcard, Acc) -> 'match_#'(Topic) ++ %% try match foo/bar/# - lookup_topic(Topic) ++ %% try match foo/bar + lookup_topic(Topic, IsWildcard) ++ %% try match foo/bar Acc; -match_compact([Word | Words], Prefix, Acc0) -> +match_compact([Word | Words], Prefix, IsWildcard, Acc0) -> Acc1 = 'match_#'(Prefix) ++ Acc0, - Acc = match_compact(Words, join(Prefix, Word), Acc1), + Acc = match_compact(Words, join(Prefix, Word), IsWildcard, Acc1), WildcardPrefix = join(Prefix, '+'), %% go deeper to match current_prefix/+ only when: %% 1. current word is the last %% OR %% 2. there is a prefix = 'current_prefix/+' case Words =:= [] orelse has_prefix(WildcardPrefix) of - true -> match_compact(Words, WildcardPrefix, Acc); + true -> match_compact(Words, WildcardPrefix, true, Acc); false -> Acc end. diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 23cd1009f..7ae23b4c6 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -102,11 +102,11 @@ t_match2(_) -> ?assertEqual([], ?TRIE:match(<<"$SYS/broker/zenmq">>)). t_match3(_) -> - Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], + Topics = [<<"d/#">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], trans(fun() -> [emqx_trie:insert(Topic) || Topic <- Topics] end), Matched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"a/b/c">>]), case length(Matched) of - 4 -> ok; + 3 -> ok; _ -> error({unexpected, Matched}) end, SysMatched = emqx_trie:match(<<"$SYS/a/b/c">>), From 78fd1a80c5348f1f5dba4d13d8fafdf00bf676d7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 18:32:06 +0200 Subject: [PATCH 36/72] fix(bench): test more publish levels --- src/emqx_broker_bench.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_broker_bench.erl b/src/emqx_broker_bench.erl index 725a65898..b46799eae 100644 --- a/src/emqx_broker_bench.erl +++ b/src/emqx_broker_bench.erl @@ -30,7 +30,7 @@ run1(Subs, SubOps, Pubs, PubOps) -> sub_ops => SubOps, pub_ops => PubOps, sub_ptn => <<"device/{{id}}/+/{{num}}/#">>, - pub_ptn => <<"device/{{id}}/foo/{{num}}/bar">> + pub_ptn => <<"device/{{id}}/foo/{{num}}/bar/1/2/3/4/5">> }). %% setting fields: From ed0ad3e796f8102b41198e57c9b603ca4fe4adaa Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 18:33:54 +0200 Subject: [PATCH 37/72] chore: skip printing memory --- src/emqx_broker_bench.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/emqx_broker_bench.erl b/src/emqx_broker_bench.erl index b46799eae..5aad43cc9 100644 --- a/src/emqx_broker_bench.erl +++ b/src/emqx_broker_bench.erl @@ -72,7 +72,6 @@ run(#{subscribers := Subs, io:format(user, "LookupRps: ~p~n", [rps(Pubs * PubOps, T2)]), io:format(user, "mnesia table(s) RAM: ~p~n", [ram_bytes()]), - io:format(user, "erlang memory: ~p~n", [erlang:memory()]), io:format(user, "unsubscribe ...~n", []), {T3, ok} = From 9698933f83879de5ef4151af34141483870c925a Mon Sep 17 00:00:00 2001 From: Rory-Z Date: Fri, 14 May 2021 09:21:10 +0000 Subject: [PATCH 38/72] chore(CI): ignore changes to the etc directory when checking the app version --- scripts/apps-version-check.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 49350b007..098653444 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -16,7 +16,6 @@ while read -r app; do now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') if [ "$old_app_version" = "$now_app_version" ]; then changed="$(git diff --name-only "$latest_release"...HEAD \ - -- "$app_path/etc" \ -- "$app_path/src" \ -- "$app_path/priv" \ -- "$app_path/c_src" | wc -l)" From c6771a57b636cb43b92f33db4288648fb31e1b80 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 14 May 2021 14:03:33 +0800 Subject: [PATCH 39/72] chore(update dep): update dep of ehttpc --- apps/emqx_web_hook/src/emqx_web_hook.erl | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl index 7af83d749..b13b87fbe 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook.erl @@ -338,7 +338,7 @@ send_http_request(ClientID, Params) -> ?LOG(warning, "HTTP request failed with status code: ~p", [StatusCode]), ok; {error, Reason} -> - ?LOG(error, "HTTP request error: ~p", [Reason]), + ?LOG(error, "HTTP request path ~p error: ~p", [Path, Reason]), ok end. diff --git a/rebar.config b/rebar.config index 90841fbfa..2e5a27a85 100644 --- a/rebar.config +++ b/rebar.config @@ -35,7 +35,7 @@ {erl_first_files, ["src/emqx_logger.erl", "src/emqx_rule_actions_trans.erl"]}. {deps, - [ {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.3"}}} + [ {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.4"}}} , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 09f51e51bd1adfb841a26d892ea0fb08ae95a815 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 14 May 2021 16:35:22 +0800 Subject: [PATCH 40/72] chore(update dep): update right tag for ehttpc --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2e5a27a85..338def9c1 100644 --- a/rebar.config +++ b/rebar.config @@ -35,7 +35,7 @@ {erl_first_files, ["src/emqx_logger.erl", "src/emqx_rule_actions_trans.erl"]}. {deps, - [ {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.4"}}} + [ {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.5"}}} , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 3984f8cd2f27064974f6deca2f5daea5e616b79f Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 14 May 2021 18:28:25 +0800 Subject: [PATCH 41/72] chore(log): update log content --- apps/emqx_web_hook/src/emqx_web_hook.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl index b13b87fbe..7af83d749 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook.erl @@ -338,7 +338,7 @@ send_http_request(ClientID, Params) -> ?LOG(warning, "HTTP request failed with status code: ~p", [StatusCode]), ok; {error, Reason} -> - ?LOG(error, "HTTP request path ~p error: ~p", [Path, Reason]), + ?LOG(error, "HTTP request error: ~p", [Reason]), ok end. From ff4ac624d07a6bbf9ef01f73db8cd109b9aaa7c8 Mon Sep 17 00:00:00 2001 From: Rory-Z Date: Fri, 14 May 2021 11:28:12 +0000 Subject: [PATCH 42/72] chore(release): update emqx release version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index cc5064c69..2b490d6d8 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.0"}). +-define(EMQX_RELEASE, {opensource, "4.3.1"}). -else. From f21ef5093898df5904dec08c2935f6ef2c91d820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20Skirgaj=C5=82=C5=82o?= Date: Fri, 14 May 2021 22:40:51 +0200 Subject: [PATCH 43/72] chore: add missing parameter in available variables --- apps/emqx_auth_http/etc/emqx_auth_http.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 56a483681..058c86dbf 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -96,6 +96,7 @@ auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded ## When the request method is POST, the final format is determined by content-type ## ## Available Variables: +## - %A: access (1 - subscribe, 2 - publish) ## - %u: username ## - %c: clientid ## - %a: ipaddress @@ -104,6 +105,7 @@ auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded ## - %p: sockport of server accepted ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert +## - %t: topic ## ## Value: =,=,... auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m From a7d2f44a5bf6463391503f4d0f6a9ec259a09f68 Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 17 May 2021 13:23:47 +0800 Subject: [PATCH 44/72] chore(optimize): optimize MGMT test cases --- ...x_bridge_mqtt_data_export_import_SUITE.erl | 4 +- .../test/emqx_mgmt_api_SUITE.erl | 88 +++++++------------ .../emqx_webhook_data_export_import_SUITE.erl | 17 +--- 3 files changed, 38 insertions(+), 71 deletions(-) diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index aca60a687..bae633045 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -28,7 +28,9 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - emqx_ct_helpers:start_apps([emqx_bridge_mqtt, emqx_rule_engine]), + application:load(emqx_modules), + application:load(emqx_bridge_mqtt), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]), Cfg. end_per_suite(Cfg) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 86b65939c..1ba958cc5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -34,55 +34,22 @@ -define(BASE_PATH, "api"). all() -> - [{group, rest_api}]. - -groups() -> - [{rest_api, - [sequence], - [ alarms - , apps - , banned - , brokers - , clients - , listeners - , metrics - , nodes - , plugins - , acl_cache - , pubsub - , routes_and_subscriptions - , stats - , data - ] - }]. + emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia, emqx_modules]), - ekka_mnesia:start(), - emqx_mgmt_auth:mnesia(boot), + application:load(emqx_modules), + emqx_ct_helpers:start_apps([emqx_management]), Config. -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx_modules]), - ekka_mnesia:ensure_stopped(). - -init_per_testcase(data, Config) -> - ok = emqx_dashboard_admin:mnesia(boot), - application:ensure_all_started(emqx_dashboard), - ok = emqx_rule_registry:mnesia(boot), - application:ensure_all_started(emqx_rule_engine), - Config; +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_management]), + Config. init_per_testcase(_, Config) -> Config. -end_per_testcase(data, _Config) -> - application:stop(emqx_dahboard), - application:stop(emqx_rule_engine), - ok; - -end_per_testcase(_, _Config) -> - ok. +end_per_testcase(_, Config) -> + Config. get(Key, ResponseBody) -> maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). @@ -101,7 +68,7 @@ is_existing(Name, [_Alarm | More]) -> is_existing(_Name, []) -> false. -alarms(_) -> +t_alarms(_) -> emqx_alarm:activate(alarm1), emqx_alarm:activate(alarm2), @@ -134,7 +101,7 @@ alarms(_) -> ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). -apps(_) -> +t_apps(_) -> AppId = <<"123456">>, meck:new(emqx_mgmt_auth, [passthrough, no_history]), meck:expect(emqx_mgmt_auth, add_app, 6, fun(_, _, _, _, _, _) -> {error, undefined} end), @@ -172,7 +139,7 @@ apps(_) -> [App] = get(<<"data">>, Result), ?assertEqual(<<"admin">>, maps:get(<<"app_id">>, App)). -banned(_) -> +t_banned(_) -> Who = <<"myclient">>, {ok, _} = request_api(post, api_path(["banned"]), [], auth_header_(), #{<<"who">> => Who, @@ -190,7 +157,7 @@ banned(_) -> {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), ?assertEqual([], get(<<"data">>, Result2)). -brokers(_) -> +t_brokers(_) -> {ok, _} = request_api(get, api_path(["brokers"]), auth_header_()), {ok, _} = request_api(get, api_path(["brokers", atom_to_list(node())]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), @@ -199,7 +166,7 @@ brokers(_) -> ?assertEqual(<<"undefined">>, get(<<"message">>, Error)), meck:unload(emqx_mgmt). -clients(_) -> +t_clients(_) -> process_flag(trap_exit, true), Username1 = <<"user1">>, Username2 = <<"user2">>, @@ -288,7 +255,7 @@ receive_exit(Count) -> ct:log("timeout") end. -listeners(_) -> +t_listeners(_) -> {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), @@ -299,7 +266,7 @@ listeners(_) -> maps:get(<<"error">>, maps:get(<<"listeners">>, Error))), meck:unload(emqx_mgmt). -metrics(_) -> +t_metrics(_) -> {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), @@ -307,7 +274,7 @@ metrics(_) -> {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:unload(emqx_mgmt). -nodes(_) -> +t_nodes(_) -> {ok, _} = request_api(get, api_path(["nodes"]), auth_header_()), {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node())]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), @@ -317,7 +284,8 @@ nodes(_) -> ?assertEqual(<<"undefined">>, maps:get(<<"error">>, Error)), meck:unload(emqx_mgmt). -plugins(_) -> +t_plugins(_) -> + application:ensure_all_started(emqx_auth_mnesia), {ok, Plugins1} = request_api(get, api_path(["plugins"]), auth_header_()), [Plugins11] = filter(get(<<"data">>, Plugins1), <<"node">>, atom_to_binary(node(), utf8)), [Plugin1] = filter(maps:get(<<"plugins">>, Plugins11), <<"name">>, <<"emqx_auth_mnesia">>), @@ -370,9 +338,10 @@ plugins(_) -> atom_to_list(emqx_auth_mnesia), "unload"]), auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)). + ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), + application:stop(emqx_auth_mnesia). -acl_cache(_) -> +t_acl_cache(_) -> ClientId = <<"client1">>, Topic = <<"mytopic">>, {ok, C1} = emqtt:start_link(#{clientid => ClientId}), @@ -395,7 +364,7 @@ acl_cache(_) -> ?assertEqual(0, length(Caches3)), ok = emqtt:disconnect(C1). -pubsub(_) -> +t_pubsub(_) -> Qos1Received = emqx_metrics:val('messages.qos1.received'), Qos2Received = emqx_metrics:val('messages.qos2.received'), Received = emqx_metrics:val('messages.received'), @@ -514,7 +483,7 @@ loop(Data) -> ?assertEqual(0, maps:get(<<"code">>, H)), loop(T). -routes_and_subscriptions(_) -> +t_routes_and_subscriptions(_) -> ClientId = <<"myclient">>, Topic = <<"mytopic">>, {ok, NonRoute} = request_api(get, api_path(["routes"]), auth_header_()), @@ -559,7 +528,7 @@ routes_and_subscriptions(_) -> ok = emqtt:disconnect(C1). -stats(_) -> +t_stats(_) -> {ok, _} = request_api(get, api_path(["stats"]), auth_header_()), {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), @@ -568,7 +537,11 @@ stats(_) -> ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), meck:unload(emqx_mgmt). -data(_) -> +t_data(_) -> + ok = emqx_rule_registry:mnesia(boot), + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + application:ensure_all_started(emqx_dashboard), {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), @@ -576,7 +549,8 @@ data(_) -> ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), - + application:stop(emqx_rule_engine), + application:stop(emqx_dahboard), ok. request_api(Method, Url, Auth) -> diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl index 03d348d2a..2965b7ad0 100644 --- a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl @@ -28,24 +28,15 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - emqx_ct_helpers:start_apps([emqx_web_hook, - emqx_bridge_mqtt, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard]), - ok = ekka_mnesia:start(), + application:load(emqx_modules), + application:load(emqx_web_hook), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]), ok = emqx_rule_registry:mnesia(boot), ok = emqx_rule_engine:load_providers(), Cfg. end_per_suite(Cfg) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, - emqx_management, - emqx_modules, - emqx_rule_engine, - emqx_bridge_mqtt, - emqx_web_hook]), + emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. get_data_path() -> From 18036fdf3bf3a86ed79846172ffd11f0ec4f7c90 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 17 May 2021 10:39:04 +0800 Subject: [PATCH 45/72] chore(CI): show rebar3.crushdump when actions failed --- .github/workflows/run_test_cases.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 430e73594..07afdcde2 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -128,6 +128,9 @@ jobs: printenv > .env docker exec -i erlang bash -c "make cover" docker exec --env-file .env -i erlang bash -c "make coveralls" + - name: cat rebar.crashdump + if: failure() + run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump' fi - uses: actions/upload-artifact@v1 if: failure() with: From 5013fb6920a4540c022dbda523f7daa2f48af205 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 16 May 2021 19:22:38 +0200 Subject: [PATCH 46/72] fix(node_dump): obfuscate more secrets --- src/emqx.app.src | 2 +- src/emqx.appup.src | 7 +++++++ src/emqx_node_dump.erl | 28 ++++++++++++++++++++-------- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 449ffd311..b195d7a1b 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 2cac1e315..937f31eff 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,6 +1,9 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.1", [ + {load_module, emqx_node_dump, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, @@ -12,6 +15,9 @@ {<<".*">>, []} ], [ + {"4.3.1", [ + {load_module, emqx_node_dump, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, @@ -21,6 +27,7 @@ %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []} ]}, + {<<".*">>, []} ] }. diff --git a/src/emqx_node_dump.erl b/src/emqx_node_dump.erl index 7134684e1..18189bb57 100644 --- a/src/emqx_node_dump.erl +++ b/src/emqx_node_dump.erl @@ -45,16 +45,28 @@ censor(Path, M) when is_map(M) -> maps:map(Fun, M); censor(Path, L = [Fst|_]) when is_tuple(Fst) -> [censor(Path, I) || I <- L]; -censor(Path, Val) -> - case Path of - [password|_] -> - obfuscate_value(Val); - [secret|_] -> - obfuscate_value(Val); - _ -> - Val +censor([Key | _], Val) -> + case is_sensitive(Key) of + true -> obfuscate_value(Val); + false -> Val end. +is_sensitive(Key) when is_atom(Key) -> + is_sensitive(atom_to_binary(Key)); +is_sensitive(Key) when is_list(Key) -> + try iolist_to_binary(Key) of + Bin -> + is_sensitive(Bin) + catch + _ : _ -> + false + end; +is_sensitive(Key) when is_binary(Key) -> + lists:any(fun(Pattern) -> re:run(Key, Pattern) =/= nomatch end, + ["passwd", "password", "secret"]); +is_sensitive(Key) when is_tuple(Key) -> + false. + obfuscate_value(Val) when is_binary(Val) -> <<"********">>; obfuscate_value(_Val) -> From 4cd056cab54d61b328bb1ce2fb0a947ea1ae7a8d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 17 May 2021 17:59:01 +0800 Subject: [PATCH 47/72] fix(rule_engine): update status to false when refresh resource failed (#4821) * fix(rule_engine): update status to false when refresh resource failed * fix(dialyzer): ignore import_modules/1 created fun has no local return * fix(appup): update appup for rule engine --- .../src/emqx_mgmt_data_backup.erl | 1 + .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 14 +++++-- .../emqx_rule_engine/src/emqx_rule_engine.erl | 37 +++++++++---------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 1eb6f8245..71a92686e 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -503,6 +503,7 @@ do_import_acl_mnesia(Acls) -> end. -ifdef(EMQX_ENTERPRISE). +-dialyzer({nowarn_function, [import_modules/1]}). import_modules(Modules) -> case ets:info(emqx_modules) of undefined -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 86846ccde..83b2d7632 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 17cecac68..446e082b7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,13 +1,21 @@ %% -*-: erlang -*- -{"4.3.1", +{"4.3.2", [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, + {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + ]}, + {"4.3.1", + [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, + {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + ]}, + {"4.3.1", + [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 07827ed1f..c2ccf2c29 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -408,7 +408,7 @@ refresh_resource_status() -> fun(#resource{id = ResId, type = ResType}) -> case emqx_rule_registry:find_resource_type(ResType) of {ok, #resource_type{on_status = {Mod, OnStatus}}} -> - fetch_resource_status(Mod, OnStatus, ResId); + _ = fetch_resource_status(Mod, OnStatus, ResId); _ -> ok end end, emqx_rule_registry:get_resources()). @@ -588,27 +588,26 @@ clear_action(Module, Destroy, ActionInstId) -> fetch_resource_status(Module, OnStatus, ResId) -> case emqx_rule_registry:find_resource_params(ResId) of {ok, ResParams = #resource_params{params = Params, status = #{is_alive := LastIsAlive}}} -> - try - NewStatus = - case Module:OnStatus(ResId, Params) of - #{is_alive := LastIsAlive} = Status -> Status; - #{is_alive := true} = Status -> - {ok, Type} = find_type(ResId), - Name = alarm_name_of_resource_down(Type, ResId), - emqx_alarm:deactivate(Name), - Status; - #{is_alive := false} = Status -> - {ok, Type} = find_type(ResId), - Name = alarm_name_of_resource_down(Type, ResId), - emqx_alarm:activate(Name, #{id => ResId, type => Type}), - Status - end, - emqx_rule_registry:add_resource_params(ResParams#resource_params{status = NewStatus}), - NewStatus + NewStatus = try + case Module:OnStatus(ResId, Params) of + #{is_alive := LastIsAlive} = Status -> Status; + #{is_alive := true} = Status -> + {ok, Type} = find_type(ResId), + Name = alarm_name_of_resource_down(Type, ResId), + emqx_alarm:deactivate(Name), + Status; + #{is_alive := false} = Status -> + {ok, Type} = find_type(ResId), + Name = alarm_name_of_resource_down(Type, ResId), + emqx_alarm:activate(Name, #{id => ResId, type => Type}), + Status + end catch _Error:Reason:STrace -> ?LOG(error, "get resource status for ~p failed: ~0p", [ResId, {Reason, STrace}]), #{is_alive => false} - end; + end, + emqx_rule_registry:add_resource_params(ResParams#resource_params{status = NewStatus}), + NewStatus; not_found -> #{is_alive => false} end. From 97e27faf05a2c24a6115e0c06f5f592b9d1a8f28 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 18 May 2021 10:34:23 +0800 Subject: [PATCH 48/72] fix(emqx): check if the congestion alarm was sent before clearing (#4824) --- src/emqx.appup.src | 4 ++++ src/emqx_congestion.erl | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 937f31eff..7e21ef4ed 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,10 +2,12 @@ {VSN, [ {"4.3.1", [ + {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, {load_module, emqx_trie, brutal_purge, soft_purge, []}, @@ -16,11 +18,13 @@ ], [ {"4.3.1", [ + {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, {load_module, emqx_connection, brutal_purge, soft_purge, []}, + {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, {load_module, emqx_trie, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' diff --git a/src/emqx_congestion.erl b/src/emqx_congestion.erl index ea99a63b1..4ec20034d 100644 --- a/src/emqx_congestion.erl +++ b/src/emqx_congestion.erl @@ -48,7 +48,10 @@ maybe_alarm_conn_congestion(Socket, Transport, Channel) -> cancel_alarms(Socket, Transport, Channel) -> lists:foreach(fun(Reason) -> - do_cancel_alarm_congestion(Socket, Transport, Channel, Reason) + case has_alarm_sent(Reason) of + true -> do_cancel_alarm_congestion(Socket, Transport, Channel, Reason); + false -> ok + end end, ?ALL_ALARM_REASONS). is_alarm_enabled(Channel) -> From dfab9492c600360216bebefb6ff36129d03adba9 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 18 May 2021 09:48:12 +0200 Subject: [PATCH 49/72] fix(emqx_connection): do not log einval error einval happens when trying to send data to a closed socket. --- src/emqx.appup.src | 2 ++ src/emqx_connection.erl | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 7e21ef4ed..ce5ddb08e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,6 +2,7 @@ {VSN, [ {"4.3.1", [ + {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, @@ -18,6 +19,7 @@ ], [ {"4.3.1", [ + {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 89bb06e12..8b4940308 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -683,7 +683,10 @@ handle_info(activate_socket, State = #state{sockstate = OldSst}) -> end; handle_info({sock_error, Reason}, State) -> - Reason =/= closed andalso ?LOG(error, "Socket error: ~p", [Reason]), + case Reason =/= closed andalso Reason =/= einval of + true -> ?LOG(warning, "socket_error: ~p", [Reason]); + false -> ok + end, handle_info({sock_closed, Reason}, close_socket(State)); handle_info(Info, State) -> From d4dcf94da69e0fd8bb63574690e8eecb86a5b5a5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 18 May 2021 14:10:01 +0200 Subject: [PATCH 50/72] fix(emqx.cmd): windows boot --- bin/emqx.cmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index bef7da355..8b9686462 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -16,6 +16,7 @@ :: Set variables that describe the release @set rel_name=emqx @set rel_vsn={{ release_version }} +@set REL_VSN=%rel_vsn% @set erts_vsn={{ erts_vsn }} @set erl_opts={{ erl_opts }} @@ -30,6 +31,7 @@ set rel_root_dir=%%~fA ) @set rel_dir=%rel_root_dir%\releases\%rel_vsn% +@set RUNNER_ROOT_DIR=%rel_root_dir% @set etc_dir=%rel_root_dir%\etc @set lib_dir=%rel_root_dir%\lib From cace9341a06bd0e59b1981102da1414bd15fb459 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 May 2021 14:38:48 +0800 Subject: [PATCH 51/72] test(auth-jwt): fix verify_claims not being covered --- apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index f123d1037..d0a4a34a0 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -37,11 +37,11 @@ groups() -> ]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx, emqx_auth_jwt], fun set_special_configs/1), + emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_jwt, emqx]). + emqx_ct_helpers:stop_apps([emqx_auth_jwt]). set_special_configs(emqx) -> application:set_env(emqx, allow_anonymous, false), @@ -97,6 +97,8 @@ t_check_auth(_) -> t_check_claims(_) -> application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]), + application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt), + Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Jwt = sign([{client_id, <<"client1">>}, {username, <<"plain">>}, @@ -113,8 +115,9 @@ t_check_claims(_) -> t_check_claims_clientid(_) -> application:set_env(emqx_auth_jwt, verify_claims, [{clientid, <<"%c">>}]), + application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt), Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external}, - Jwt = sign([{client_id, <<"client23">>}, + Jwt = sign([{clientid, <<"client23">>}, {username, <<"plain">>}, {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), @@ -128,6 +131,8 @@ t_check_claims_clientid(_) -> t_check_claims_username(_) -> application:set_env(emqx_auth_jwt, verify_claims, [{username, <<"%u">>}]), + application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt), + Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external}, Jwt = sign([{client_id, <<"client23">>}, {username, <<"plain">>}, From 4e1798e3f3d77cddce1425a2a4aea13fb8522608 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 19 May 2021 22:31:49 +0200 Subject: [PATCH 52/72] fix(emqx_cm): do not log noproc as error 1. websocket call exit with noproc reason. 2. do not capture stacktrace when no need for it --- src/emqx.appup.src | 2 ++ src/emqx_cm.erl | 32 ++++++++++++++++++++------------ test/emqx_cm_SUITE.erl | 13 +++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index ce5ddb08e..ef6764241 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,6 +3,7 @@ [ {"4.3.1", [ {load_module, emqx_connection, brutal_purge, soft_purge, []}, + {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, @@ -20,6 +21,7 @@ [ {"4.3.1", [ {load_module, emqx_connection, brutal_purge, soft_purge, []}, + {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []} ]}, diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 7f324e322..cf505a873 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -22,6 +22,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[CM]"). @@ -279,18 +280,25 @@ takeover_session(ClientId, ChanPid) -> discard_session(ClientId) when is_binary(ClientId) -> case lookup_channels(ClientId) of [] -> ok; - ChanPids -> - lists:foreach( - fun(ChanPid) -> - try - discard_session(ClientId, ChanPid) - catch - _:{noproc,_}:_Stk -> ok; - _:{{shutdown,_},_}:_Stk -> ok; - _:Error:_Stk -> - ?LOG(error, "Failed to discard ~0p: ~0p", [ChanPid, Error]) - end - end, ChanPids) + ChanPids -> lists:foreach(fun(Pid) -> do_discard_session(ClientId, Pid) end, ChanPids) + end. + +do_discard_session(ClientId, Pid) -> + try + discard_session(ClientId, Pid) + catch + _ : noproc -> % emqx_ws_connection: call + ?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {noproc, _} -> % emqx_connection: gen_server:call + ?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {{shutdown, _}, _} -> + ?tp(debug, "session_already_shutdown", #{pid => Pid}), + ok; + _ : Error : St -> + ?tp(error, "failed_to_discard_session", + #{pid => Pid, reason => Error, stacktrace=>St}) end. discard_session(ClientId, ChanPid) when node(ChanPid) == node() -> diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index b8e0f9066..0cfba4737 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(CM, emqx_cm). -define(ChanInfo,#{conninfo => @@ -179,6 +180,18 @@ t_discard_session(_) -> ok = emqx_cm:unregister_channel(<<"clientid">>), ok = meck:unload(emqx_connection). +t_discard_session_race(_) -> + ok = snabbkaffe:start_trace(), + #{conninfo := ConnInfo0} = ?ChanInfo, + ConnInfo = ConnInfo0#{conn_mod := emqx_ws_connection}, + {Pid, Ref} = spawn_monitor(fun() -> receive stop -> exit(normal) end end), + ok = emqx_cm:register_channel(<<"clientid">>, Pid, ConnInfo), + Pid ! stop, + receive {'DOWN', Ref, process, Pid, normal} -> ok end, + ok = emqx_cm:discard_session(<<"clientid">>), + {ok, _} = ?block_until(#{?snk_kind := "session_already_gone", pid := Pid}, 1000), + snabbkaffe:stop(). + t_takeover_session(_) -> #{conninfo := ConnInfo} = ?ChanInfo, {error, not_found} = emqx_cm:takeover_session(<<"clientid">>), From c07fe0696b4b51e5c634d2424d39bf3efc9f187f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 09:03:40 +0200 Subject: [PATCH 53/72] fix(appup): ensure version bump for emqx_management --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_management.appup.src | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fe65052cf..3604d3505 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 5048e4f0f..9ad23381e 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,12 +1,12 @@ %% -*-: erlang -*- -{"4.3.1", - [ {"4.3.0", +{"4.3.2", + [ {<<"4.3.[0-1]">>, [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} ]} ], [ - {"4.3.0", + {<<"4.3.[0-1]">>, [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} ]} ] -}. \ No newline at end of file +}. From 48c1d0983d17cb901647f84c6e3885978e95ffc5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 09:24:39 +0200 Subject: [PATCH 54/72] chore(emqx_ctl): do not print application --- apps/emqx_management/src/emqx_mgmt_cli.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 252ac3857..77fe96182 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -116,13 +116,7 @@ mgmt(_) -> status([]) -> {InternalStatus, _ProvidedStatus} = init:get_status(), - emqx_ctl:print("Node ~p ~s is ~p~n", [node(), emqx_app:get_release(), InternalStatus]), - case lists:keysearch(?APP, 1, application:which_applications()) of - false -> - emqx_ctl:print("Application ~s is not running~n", [?APP]); - {value, {?APP, _Desc, Vsn}} -> - emqx_ctl:print("Application ~s ~s is running~n", [?APP, Vsn]) - end; + emqx_ctl:print("Node ~p ~s is ~p~n", [node(), emqx_app:get_release(), InternalStatus]); status(_) -> emqx_ctl:usage("status", "Show broker status"). From 5fe395cee52f51fdb2d47d5fa81adbc532cfbe16 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 10:25:04 +0200 Subject: [PATCH 55/72] test: fix emqx_ctl printout test --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 42578baf7..6bac9b4c7 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -133,7 +133,8 @@ t_mgmt_cmd(_) -> t_status_cmd(_) -> % ct:pal("start testing status command"), mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "is running")), + %% init internal status seem to be always 'starting' when running ct tests + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), meck:unload(). t_broker_cmd(_) -> From ce2739ef3d6a461a28922818f82bdb81ccf9dfe9 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 10:38:39 +0200 Subject: [PATCH 56/72] chore(emqx_management): update appup --- apps/emqx_management/src/emqx_management.appup.src | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 9ad23381e..3206ce31b 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -2,11 +2,13 @@ {"4.3.2", [ {<<"4.3.[0-1]">>, [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + , {load_module, emqx_mgmt_cli, brutal_purge, soft_purge, []} ]} ], [ {<<"4.3.[0-1]">>, [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + , {load_module, emqx_mgmt_cli, brutal_purge, soft_purge, []} ]} ] }. From e7e87837e5c833f84eb71442235ea633c3eaebd6 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 11:07:50 +0200 Subject: [PATCH 57/72] test: fix emqx_ctl status grep pattern --- .ci/build_packages/tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 1f9ca2651..bbc107d37 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -31,7 +31,7 @@ emqx_test(){ echo "running ${packagename} start" "${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while [ -z "$("${PACKAGE_PATH}"/emqx/bin/emqx_ctl status |grep 'is running'|awk '{print $1}')" ] + while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do if [ $IDLE_TIME -gt 10 ] then @@ -103,7 +103,7 @@ running_test(){ emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while [ -z "$(emqx_ctl status |grep 'is running'|awk '{print $1}')" ] + while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do if [ $IDLE_TIME -gt 10 ] then @@ -121,7 +121,7 @@ running_test(){ || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while [ -z "$(emqx_ctl status |grep 'is running'|awk '{print $1}')" ] + while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted' do if [ $IDLE_TIME -gt 10 ] then From e8790f6e111c61aa54665a7112b4b427f5ae2d23 Mon Sep 17 00:00:00 2001 From: Turtle Date: Fri, 21 May 2021 14:07:37 +0800 Subject: [PATCH 58/72] fix(mqtt-sn): Fix stop due to keepAlive at sleep mode --- apps/emqx_sn/src/emqx_sn.app.src | 2 +- apps/emqx_sn/src/emqx_sn.appup.src | 17 +++++++++++++++++ apps/emqx_sn/src/emqx_sn_asleep_timer.erl | 15 ++++++++++----- apps/emqx_sn/src/emqx_sn_gateway.erl | 18 ++++++++++-------- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 10 +++++----- src/emqx.appup.src | 15 ++++++++++++--- src/emqx_channel.erl | 13 ++++++++++++- 7 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 apps/emqx_sn/src/emqx_sn.appup.src diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index b0453c774..3d7db1b02 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src new file mode 100644 index 000000000..499664fe1 --- /dev/null +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -0,0 +1,17 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {load_module, emqx_sn_asleep_timer, brutal_purge, soft_purge, []}, + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, [emqx_sn_asleep_timer]} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {load_module, emqx_sn_asleep_timer, brutal_purge, soft_purge, []}, + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, [emqx_sn_asleep_timer]} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl b/apps/emqx_sn/src/emqx_sn_asleep_timer.erl index 56a63ee2f..37ea67689 100644 --- a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl +++ b/apps/emqx_sn/src/emqx_sn_asleep_timer.erl @@ -18,6 +18,7 @@ -export([ init/0 , ensure/2 + , cancel/1 ]). -record(asleep_state, { @@ -42,8 +43,8 @@ init() -> -spec(ensure(undefined | integer(), asleep_state()) -> asleep_state()). ensure(undefined, State = #asleep_state{duration = Duration}) -> ensure(Duration, State); -ensure(Duration, State = #asleep_state{tref = TRef}) -> - _ = cancel(TRef), +ensure(Duration, State) -> + cancel(State), State#asleep_state{duration = Duration, tref = start(Duration)}. %%-------------------------------------------------------------------- @@ -55,6 +56,10 @@ ensure(Duration, State = #asleep_state{tref = TRef}) -> start(Duration) -> erlang:send_after(timer:seconds(Duration), self(), asleep_timeout). -cancel(undefined) -> ok; -cancel(TRef) when is_reference(TRef) -> - erlang:cancel_timer(TRef). +cancel(#asleep_state{tref = Timer}) when is_reference(Timer) -> + case erlang:cancel_timer(Timer) of + false -> + receive {timeout, Timer, _} -> ok after 0 -> ok end; + _ -> ok + end; +cancel(_) -> ok. \ No newline at end of file diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index bfb2f28df..96f849974 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -439,12 +439,11 @@ asleep(cast, {incoming, ?SN_PUBREC_MSG(PubRec, MsgId)}, State) % 4) emq-sn regard this CONNECT as a signal to connected state, not a bootup CONNECT. For this reason, will procedure is lost % this should be a bug in mqtt-sn channel. asleep(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, - State = #state{keepalive_interval = _Interval}) -> - % device wakeup and goto connected state - % keepalive timer may timeout in asleep state and delete itself, need to restart keepalive - % TODO: Fixme later. - %% self() ! {keepalive, start, Interval}, - {next_state, connected, send_connack(State)}; + State = #state{channel = Channel, asleep_timer = Timer}) -> + NChannel = emqx_channel:ensure_keepalive(#{}, Channel), + emqx_sn_asleep_timer:cancel(Timer), + {next_state, connected, send_connack(State#state{channel = NChannel, + asleep_timer = emqx_sn_asleep_timer:init()})}; asleep(EventType, EventContent, State) -> handle_event(EventType, EventContent, asleep, State). @@ -771,10 +770,13 @@ send_message(Msg = #mqtt_sn_message{type = Type}, goto_asleep_state(State) -> goto_asleep_state(undefined, State). -goto_asleep_state(Duration, State=#state{asleep_timer = AsleepTimer}) -> +goto_asleep_state(Duration, State=#state{asleep_timer = AsleepTimer, + channel = Channel}) -> ?LOG(debug, "goto_asleep_state Duration=~p", [Duration]), NewTimer = emqx_sn_asleep_timer:ensure(Duration, AsleepTimer), - {next_state, asleep, State#state{asleep_timer = NewTimer}, hibernate}. + NChannel = emqx_channel:clear_keepalive(Channel), + {next_state, asleep, State#state{asleep_timer = NewTimer, + channel = NChannel}, hibernate}. %%-------------------------------------------------------------------- %% Helper funcs diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index ad0c5f032..2972571be 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -856,7 +856,7 @@ t_will_test2(_) -> send_pingreq_msg(Socket, undefined), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - timer:sleep(10000), + timer:sleep(4000), receive_response(Socket), % ignore PUBACK receive_response(Socket), % ignore PUBCOMP @@ -878,7 +878,7 @@ t_will_test3(_) -> send_pingreq_msg(Socket, undefined), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), - timer:sleep(10000), + timer:sleep(4000), ?assertEqual(udp_receive_timeout, receive_response(Socket)), @@ -906,7 +906,7 @@ t_will_test4(_) -> send_willmsgupd_msg(Socket, <<"1A2B3C">>), ?assertEqual(<<3, ?SN_WILLMSGRESP, ?SN_RC_ACCEPTED>>, receive_response(Socket)), - timer:sleep(10000), + timer:sleep(4000), receive_response(Socket), % ignore PUBACK @@ -1359,7 +1359,7 @@ t_asleep_test07_to_connected(_) -> timer:sleep(1500), % asleep timer should get timeout, without any effect - timer:sleep(9000), + timer:sleep(4000), % keepalive timer should get timeout gen_udp:close(Socket). @@ -1517,7 +1517,7 @@ t_awake_test01_to_connected(_) -> timer:sleep(1500), % asleep timer should get timeout - timer:sleep(9000), + timer:sleep(4000), % keepalive timer should get timeout gen_udp:close(Socket). diff --git a/src/emqx.appup.src b/src/emqx.appup.src index ef6764241..bde6af882 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -5,7 +5,8 @@ {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, - {load_module, emqx_node_dump, brutal_purge, soft_purge, []} + {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, + {load_module, emqx_channel, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -13,6 +14,9 @@ {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, {load_module, emqx_trie, brutal_purge, soft_purge, []}, + {load_module, emqx_cm, brutal_purge, soft_purge, []}, + {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, + {load_module, emqx_channel, brutal_purge, soft_purge, []}, {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, @@ -23,7 +27,8 @@ {load_module, emqx_connection, brutal_purge, soft_purge, []}, {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, - {load_module, emqx_node_dump, brutal_purge, soft_purge, []} + {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, + {load_module, emqx_channel, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -31,9 +36,13 @@ {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_frame, brutal_purge, soft_purge, []}, {load_module, emqx_trie, brutal_purge, soft_purge, []}, + {load_module, emqx_cm, brutal_purge, soft_purge, []}, + {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, + {load_module, emqx_channel, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. - {load_module, emqx_metrics, brutal_purge, soft_purge, []} + {load_module, emqx_metrics, brutal_purge, soft_purge, []}, + {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, {<<".*">>, []} diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index c8acccce9..3a818b9a7 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -49,7 +49,10 @@ ]). %% Export for emqx_sn --export([do_deliver/2]). +-export([ do_deliver/2 + , ensure_keepalive/2 + , clear_keepalive/1 + ]). %% Exports for CT -export([set_field/3]). @@ -1562,6 +1565,14 @@ ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone} Keepalive = emqx_keepalive:init(round(timer:seconds(Interval) * Backoff)), ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). +clear_keepalive(Channel = #channel{timers = Timers}) -> + case maps:get(alive_timer, Timers, undefined) of + undefined -> + Channel; + TRef -> + emqx_misc:cancel_timer(TRef), + Channel#channel{timers = maps:without([alive_timer], Timers)} + end. %%-------------------------------------------------------------------- %% Maybe Resume Session From a8cb5678b56ecb59e0abba717d5ebcdd2ba9ded6 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 14:39:47 +0200 Subject: [PATCH 59/72] test: ensure erlang 23 in github action --- .github/workflows/run_fvt_tests.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index eea7acde5..baa76569a 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,6 +16,10 @@ jobs: steps: - uses: actions/checkout@v1 + - uses: gleam-lang/setup-erlang@v1.1.2 + id: install_erlang + with: + otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -52,7 +56,7 @@ jobs: output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) if [ "$expected" != "$output" ]; then exit 1 - fi + fi - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then From 7a969e085f664152cd4da617bf74b1d0bd4606f7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 21 May 2021 17:27:11 +0200 Subject: [PATCH 60/72] test: ensure erlang 23 in more github actions --- .github/workflows/run_fvt_tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index baa76569a..a847c07ca 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -70,6 +70,10 @@ jobs: steps: - uses: actions/checkout@v1 + - uses: gleam-lang/setup-erlang@v1.1.2 + id: install_erlang + with: + otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then From 1e562d4bc620e6051298e1beed26d650506dbb23 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Sat, 22 May 2021 01:38:55 +0200 Subject: [PATCH 61/72] chore(rule_engine): Don't import alias/1 BIF Fixes: #4847 --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 4c2b661ea..f9e210ab3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -31,6 +31,8 @@ , range_get/3 ]). +-compile({no_auto_import,[alias/1]}). + -type(input() :: map()). -type(alias() :: atom()). -type(collection() :: {alias(), [term()]}). From 67245b06b102ffa969707967ef8c54dbb0356d16 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 22 May 2021 08:29:06 +0200 Subject: [PATCH 62/72] chore: make sure emqx_app is reloaded we bump release version number in emqx_release.hrl for each release this version is retrieved from emqx_app module, so we must always reload this module during hot upgrade --- src/emqx.appup.src | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index bde6af882..4c649e68d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,7 +6,8 @@ {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, - {load_module, emqx_channel, brutal_purge, soft_purge, []} + {load_module, emqx_channel, brutal_purge, soft_purge, []}, + {load_module, emqx_app, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -17,6 +18,8 @@ {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, + {load_module, emqx_app, brutal_purge, soft_purge, []}, + %% {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, @@ -28,7 +31,8 @@ {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, - {load_module, emqx_channel, brutal_purge, soft_purge, []} + {load_module, emqx_channel, brutal_purge, soft_purge, []}, + {load_module, emqx_app, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -39,12 +43,12 @@ {load_module, emqx_cm, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, + {load_module, emqx_app, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} ]}, - {<<".*">>, []} ] }. From 09084e4e27366fe997fe7962d84bde84e5054ad0 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 24 May 2021 13:45:51 +0800 Subject: [PATCH 63/72] chore(CI): fix relup tests error in enterprise repo --- .github/workflows/run_fvt_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a847c07ca..8ca1dc80f 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -260,7 +260,7 @@ jobs: set -e -x -u if [ -n "$OLD_VSNS" ]; then mkdir -p packages - cp emqx/_packages/emqx/*.zip packages + cp emqx/_packages/${PROFILE}/*.zip packages cp emqx/_upgrade_base/*.zip packages lux -v \ --timeout 600000 \ From 6436217e075a591450c7d51f7779d23eb4655750 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 11:40:09 +0200 Subject: [PATCH 64/72] chore: fix wrong plugins dir --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index b5223013c..c6c9e3409 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -194,7 +194,7 @@ overlay_vars_pkg(bin) -> , {platform_etc_dir, "etc"} , {platform_lib_dir, "lib"} , {platform_log_dir, "log"} - , {platform_plugins_dir, "plugins"} + , {platform_plugins_dir, "etc/plugins"} , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"} , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} From faecde9ce196500b6e55e4971f9fe4f61e00ec42 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 13:31:10 +0200 Subject: [PATCH 65/72] fix(emqx_plugins): allow loading conf for plugin app dir Prior to this change, plugin config files are only allowed to be placed in the collective config dir etc/plugins. In order to support external plugin's drop-in deployment, this commit made emqx_plugins module to read conf file in application's etc dir --- src/emqx_plugins.erl | 103 +++++++++++------- test/emqx_plugins_SUITE.erl | 18 ++- .../emqx_mini_plugin/Makefile | 13 +-- .../emqx_mini_plugin/rebar.config | 4 +- 4 files changed, 75 insertions(+), 63 deletions(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index bf8de4a59..2616c1f70 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -61,7 +61,7 @@ init() -> %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - load_expand_plugins(), + ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)), case emqx:get_env(plugins_loaded_file) of undefined -> ignore; %% No plugins available File -> @@ -148,46 +148,61 @@ init_config(CfgFile) -> [application:set_env(App, Par, Val) || {Par, Val} <- Envs] end, AppsEnv). -load_expand_plugins() -> - case emqx:get_env(expand_plugins_dir) of - undefined -> ok; - ExpandPluginsDir -> - Plugins = filelib:wildcard("*", ExpandPluginsDir), - lists:foreach(fun(Plugin) -> - PluginDir = filename:join(ExpandPluginsDir, Plugin), +%% load external plugins which are placed in etc/plugins dir +load_ext_plugins(undefined) -> ok; +load_ext_plugins(Dir) -> + lists:foreach( + fun(Plugin) -> + PluginDir = filename:join(Dir, Plugin), case filelib:is_dir(PluginDir) of - true -> load_expand_plugin(PluginDir); + true -> load_ext_plugin(PluginDir); false -> ok end - end, Plugins) - end. + end, filelib:wildcard("*", Dir)). -load_expand_plugin(PluginDir) -> - init_expand_plugin_config(PluginDir), +load_ext_plugin(PluginDir) -> + ?LOG(debug, "loading_extra_plugin: ~s", [PluginDir]), Ebin = filename:join([PluginDir, "ebin"]), + AppFile = filename:join([Ebin, "*.app"]), + AppName = case filelib:wildcard(AppFile) of + [App] -> + list_to_atom(filename:basename(App, ".app")); + [] -> + ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]), + error({plugin_app_file_not_found, AppFile}) + end, + ok = load_plugin_app(AppName, Ebin), + ok = load_plugin_conf(AppName, PluginDir). + +load_plugin_app(AppName, Ebin) -> _ = code:add_patha(Ebin), Modules = filelib:wildcard(filename:join([Ebin, "*.beam"])), - lists:foreach(fun(Mod) -> - Module = list_to_atom(filename:basename(Mod, ".beam")), - code:load_file(Module) - end, Modules), - case filelib:wildcard(Ebin ++ "/*.app") of - [App|_] -> application:load(list_to_atom(filename:basename(App, ".app"))); - _ -> ?LOG(alert, "Plugin not found."), - {error, load_app_fail} + lists:foreach( + fun(BeamFile) -> + Module = list_to_atom(filename:basename(BeamFile, ".beam")), + case code:ensure_loaded(Module) of + {module, Module} -> ok; + {error, Reason} -> error({failed_to_load_plugin_beam, BeamFile, Reason}) + end + end, Modules), + case application:load(AppName) of + ok -> ok; + {error, {already_loaded, _}} -> ok end. -init_expand_plugin_config(PluginDir) -> - Priv = PluginDir ++ "/priv", - Etc = PluginDir ++ "/etc", - Schema = filelib:wildcard(Priv ++ "/*.schema"), - Conf = case filelib:wildcard(Etc ++ "/*.conf") of - [] -> []; - [Conf1] -> cuttlefish_conf:file(Conf1) - end, +load_plugin_conf(AppName, PluginDir) -> + Priv = filename:join([PluginDir, "priv"]), + Etc = filename:join([PluginDir, "etc"]), + Schema = filelib:wildcard(filename:join([Priv, "*.schema"])), + ConfFile = filename:join([Etc, atom_to_list(AppName) ++ ".conf"]), + Conf = case filelib:is_file(ConfFile) of + true -> cuttlefish_conf:file(ConfFile); + false -> error({conf_file_not_found, ConfFile}) + end, + ?LOG(debug, "loading_extra_plugin_config conf=~s, schema=~s", [ConfFile, Schema]), AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf), - lists:foreach(fun({AppName, Envs}) -> - [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] + lists:foreach(fun({AppName1, Envs}) -> + [application:set_env(AppName1, Par, Val) || {Par, Val} <- Envs] end, AppsEnv). ensure_file(File) -> @@ -223,19 +238,31 @@ load_plugins(Names, Persistent) -> generate_configs(App) -> ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config", - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", - SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", - case {filelib:is_file(ConfigFile), filelib:is_file(ConfFile) andalso filelib:is_file(SchemaFile)} of - {true, _} -> + case filelib:is_file(ConfigFile) of + true -> {ok, [Configs]} = file:consult(ConfigFile), Configs; - {_, true} -> + false -> + do_generate_configs(App) + end. + +do_generate_configs(App) -> + Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", + Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", + ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of + {true, _} -> Name1; + {false, true} -> Name2; + {false, false} -> error({config_not_found, [Name1, Name2]}) + end, + SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", + case filelib:is_file(SchemaFile) of + true -> Schema = cuttlefish_schema:files([SchemaFile]), Conf = cuttlefish_conf:file(ConfFile), LogFun = fun(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]) end, cuttlefish_generator:map(Schema, Conf, undefined, LogFun); - {false, false} -> - error({config_not_found, {ConfigFile, ConfFile, SchemaFile}}) + false -> + error({schema_not_found, SchemaFile}) end. apply_configs([]) -> diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 401cf87dd..f62fea5e7 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -30,24 +30,20 @@ init_per_suite(Config) -> DataPath = proplists:get_value(data_dir, Config), AppPath = filename:join([DataPath, "emqx_mini_plugin"]), - Cmd = lists:flatten(io_lib:format("cd ~s && make && cp -r etc _build/default/lib/emqx_mini_plugin/", [AppPath])), + Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])), ct:pal("Executing ~s~n", [Cmd]), ct:pal("~n ~s~n", [os:cmd(Cmd)]), - code:add_path(filename:join([AppPath, "_build", "default", "lib", "emqx_mini_plugin", "ebin"])), - put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([], fun(_) -> set_sepecial_cfg(DataPath) end), Config. - -set_sepecial_cfg(_) -> - ExpandPath = filename:dirname(code:lib_dir(emqx_mini_plugin)), +set_sepecial_cfg(PluginsDir) -> application:set_env(emqx, plugins_loaded_file, get(loaded_file)), - application:set_env(emqx, expand_plugins_dir, ExpandPath), + application:set_env(emqx, expand_plugins_dir, PluginsDir), ok. end_per_suite(_Config) -> @@ -58,7 +54,6 @@ t_load(_) -> ?assertEqual(ok, emqx_plugins:unload()), ?assertEqual({error, not_found}, emqx_plugins:load(not_existed_plugin)), - ?assertEqual({error, parse_config_file_failed}, emqx_plugins:load(emqx_mini_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), application:set_env(emqx, expand_plugins_dir, undefined), @@ -75,8 +70,9 @@ t_init_config(_) -> file:delete(ConfFile), ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)). -t_load_expand_plugin(_) -> - ?assertEqual({error, load_app_fail}, emqx_plugins:load_expand_plugin("./not_existed_path/")). +t_load_ext_plugin(_) -> + ?assertError({plugin_app_file_not_found, _}, + emqx_plugins:load_ext_plugin("./not_existed_path/")). t_list(_) -> ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()). diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile index ad02951a3..fd38ff640 100644 --- a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile +++ b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile @@ -8,6 +8,7 @@ all: compile compile: $(REBAR) compile + cp -r _build/default/lib/emqx_mini_plugin/ebin ./ clean: distclean @@ -22,14 +23,4 @@ xref: distclean: @rm -rf _build - @rm -f data/app.*.config data/vm.*.args rebar.lock - -CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish - -$(CUTTLEFISH_SCRIPT): - @${REBAR} get-deps - @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi - -app.config: $(CUTTLEFISH_SCRIPT) etc/emqx_mini_plugin.conf - $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/emqx_mini_plugin.conf -i priv/emqx_mini_plugin.schema -d data - + @rm -f ebin/ data/app.*.config data/vm.*.args rebar.lock diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config index c690b88b1..4c49da1dc 100644 --- a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config +++ b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config @@ -1,5 +1,4 @@ -{deps, - []}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -19,7 +18,6 @@ {profiles, [{test, [ {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]} ]} ]}. From 8443aa5e21c7c4fa1009c6faa1da9c12fce05c8b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 15:15:50 +0200 Subject: [PATCH 66/72] chore(typo): sepecial -> special --- apps/emqx_coap/test/emqx_coap_SUITE.erl | 6 +++--- apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl | 6 +++--- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 6 +++--- lib-ce/emqx_modules/test/emqx_modules_SUITE.erl | 4 ++-- test/emqx_plugins_SUITE.erl | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl index ed59b309c..444bcc064 100644 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -28,16 +28,16 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), Config. -set_sepecial_cfg(emqx_coap) -> +set_special_cfg(emqx_coap) -> Opts = application:get_env(emqx_coap, dtls_opts,[]), Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")}, {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}], application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)), application:set_env(emqx_coap, enable_stats, true); -set_sepecial_cfg(_) -> +set_special_cfg(_) -> ok. end_per_suite(Config) -> diff --git a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl index 7f49ece7b..1aaf6cb69 100644 --- a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl @@ -28,12 +28,12 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), Config. -set_sepecial_cfg(emqx_coap) -> +set_special_cfg(emqx_coap) -> application:set_env(emqx_coap, enable_stats, true); -set_sepecial_cfg(_) -> +set_special_cfg(_) -> ok. end_per_suite(Config) -> diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index f1fdfa9f8..70484c30e 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -55,7 +55,7 @@ metrics() -> init_per_group(GrpName, Cfg) -> put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), - emqx_ct_helpers:start_apps([emqx_exproto], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_exproto], fun set_special_cfg/1), emqx_logger:set_log_level(debug), [{servers, Svrs}, {listener_type, GrpName} | Cfg]. @@ -63,7 +63,7 @@ end_per_group(_, Cfg) -> emqx_ct_helpers:stop_apps([emqx_exproto]), emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). -set_sepecial_cfg(emqx_exproto) -> +set_special_cfg(emqx_exproto) -> LisType = get(grpname), Listeners = application:get_env(emqx_exproto, listeners, []), SockOpts = socketopts(LisType), @@ -77,7 +77,7 @@ set_sepecial_cfg(emqx_exproto) -> NListeners = [{Proto, LisType, LisOn, UpgradeOpts(Opts)} || {Proto, _Type, LisOn, Opts} <- Listeners], application:set_env(emqx_exproto, listeners, NListeners); -set_sepecial_cfg(emqx) -> +set_special_cfg(emqx) -> application:set_env(emqx, allow_anonymous, true), application:set_env(emqx, enable_acl_cache, false), ok. diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index 48bd0bd5b..dc76e8eb7 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -32,11 +32,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_cfg/1), emqx_ct_http:create_default_app(), Config. -set_sepecial_cfg(_) -> +set_special_cfg(_) -> application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), ok. diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index f62fea5e7..6d8847f43 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -37,11 +37,11 @@ init_per_suite(Config) -> put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([], fun(_) -> set_sepecial_cfg(DataPath) end), + emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end), Config. -set_sepecial_cfg(PluginsDir) -> +set_special_cfg(PluginsDir) -> application:set_env(emqx, plugins_loaded_file, get(loaded_file)), application:set_env(emqx, expand_plugins_dir, PluginsDir), ok. From 4ec10461604ef05580d594c9241ab810491ce27a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 15:47:16 +0200 Subject: [PATCH 67/72] fix: bad test case --- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 2 +- src/emqx_plugins.erl | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 1ba958cc5..bf3727ff5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -322,7 +322,7 @@ t_plugins(_) -> auth_header_()), [Plugin3] = filter(get(<<"data">>, Plugins3), <<"name">>, <<"emqx_auth_mnesia">>), ?assertEqual(<<"emqx_auth_mnesia">>, maps:get(<<"name">>, Plugin3)), - ?assertEqual(false, maps:get(<<"active">>, Plugin3)), + ?assertEqual(true, maps:get(<<"active">>, Plugin3)), {ok, _} = request_api(put, api_path(["nodes", diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 2616c1f70..f05da760e 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -225,10 +225,11 @@ filter_plugins(Names) -> end, Names). load_plugins(Names, Persistent) -> - Plugins = list(), NotFound = Names -- names(Plugins), + Plugins = list(), + NotFound = Names -- names(Plugins), case NotFound of [] -> ok; - NotFound -> ?LOG(alert, "Cannot find plugins: ~p", [NotFound]) + NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) end, NeedToLoad = Names -- NotFound -- names(started_app), lists:foreach(fun(Name) -> From 623ffc78612e3619168807bf1798516048003612 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 16:28:02 +0200 Subject: [PATCH 68/72] fix: add emqx_plugins to appup --- src/emqx.appup.src | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 4c649e68d..3bf40272c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,7 +7,8 @@ {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, - {load_module, emqx_app, brutal_purge, soft_purge, []} + {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -19,6 +20,7 @@ {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []}, %% {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} @@ -32,7 +34,8 @@ {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, - {load_module, emqx_app, brutal_purge, soft_purge, []} + {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -44,6 +47,7 @@ {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []}, From f2ccf9778091adddaf48f2cba791badbe8cc2de1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 24 May 2021 17:22:02 +0800 Subject: [PATCH 69/72] chore(CI): fix relup tests error in enterprise repo --- .ci/fvt_tests/relup.lux | 11 ++++++----- .github/workflows/run_fvt_tests.yaml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 85b47bb60..cbecb9e14 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -1,3 +1,4 @@ +[config var=PROFILE] [config var=PACKAGE_PATH] [config var=BENCH_PATH] [config var=ONE_MORE_EMQX_PATH] @@ -21,7 +22,7 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o emqx-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip + !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip ?SH-PROMPT !cd emqx @@ -33,8 +34,8 @@ [shell emqx2] !cd $PACKAGE_PATH - !cp -f $ONE_MORE_EMQX_PATH/one_more_emqx.sh . - !./one_more_emqx.sh emqx2 + !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh . + !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2 ?SH-PROMPT !cd emqx2 @@ -75,7 +76,7 @@ ???sent [shell emqx] - !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" @@ -90,7 +91,7 @@ ?SH-PROMPT [shell emqx2] - !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8ca1dc80f..12f9da95c 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -264,6 +264,7 @@ jobs: cp emqx/_upgrade_base/*.zip packages lux -v \ --timeout 600000 \ + --var PROFILE=$PROFILE \ --var PACKAGE_PATH=$(pwd)/packages \ --var BENCH_PATH=$(pwd)/emqtt-bench \ --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ From e82e24d21f23be90c633045d5dafc4e111c3502e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 24 May 2021 12:19:53 +0200 Subject: [PATCH 70/72] fix: pin cuttlefish v3.3.5 this cuttlefish release has only changes in a test module --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 338def9c1..7a1444dc1 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.4"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.5"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} From 61bd95fddeb9c0d255079c5da1a950cf58d695bd Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 24 May 2021 12:21:08 +0200 Subject: [PATCH 71/72] build: pin otp 24 for dev --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index e7e734a7b..b87853803 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-1 +erlang 24.0.1-emqx-1 From 6701d716dde6fddb4522c421d4f2752b1f7d7498 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 23 May 2021 12:10:04 +0200 Subject: [PATCH 72/72] feat: async API to support tcp keepalive inet options --- src/emqx_cm.erl | 1 + src/emqx_connection.erl | 59 +++++++++++++++++++++++++++++++++++++++- test/emqx_mqtt_SUITE.erl | 52 +++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index cf505a873..b15a2ff79 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -111,6 +111,7 @@ start_link() -> insert_channel_info(ClientId, Info, Stats) -> Chan = {ClientId, self()}, true = ets:insert(?CHAN_INFO_TAB, {Chan, Info, Stats}), + ?tp(debug, insert_channel_info, #{client_id => ClientId}), ok. %% @private diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 8b4940308..814fd9007 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -41,8 +41,13 @@ , stats/1 ]). +-export([ async_set_keepalive/4 + , async_set_socket_options/2 + ]). + -export([ call/2 , call/3 + , cast/2 ]). %% Callback @@ -56,7 +61,7 @@ ]). %% Internal callback --export([wakeup_from_hib/2, recvloop/2]). +-export([wakeup_from_hib/2, recvloop/2, get_state/1]). %% Export for CT -export([set_field/3]). @@ -184,6 +189,35 @@ stats(#state{transport = Transport, ProcStats = emqx_misc:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). +%% @doc Set TCP keepalive socket options to override system defaults. +%% Idle: The number of seconds a connection needs to be idle before +%% TCP begins sending out keep-alive probes (Linux default 7200). +%% Interval: The number of seconds between TCP keep-alive probes +%% (Linux default 75). +%% Probes: The maximum number of TCP keep-alive probes to send before +%% giving up and killing the connection if no response is +%% obtained from the other end (Linux default 9). +%% +%% NOTE: This API sets TCP socket options, which has nothing to do with +%% the MQTT layer's keepalive (PINGREQ and PINGRESP). +async_set_keepalive(Pid, Idle, Interval, Probes) -> + Options = [ {keepalive, true} + , {raw, 6, 4, <>} + , {raw, 6, 5, <>} + , {raw, 6, 6, <>} + ], + async_set_socket_options(Pid, Options). + +%% @doc Set custom socket options. +%% This API is made async because the call might be originated from +%% a hookpoint callback (otherwise deadlock). +%% If failed to set, the error message is logged. +async_set_socket_options(Pid, Options) -> + cast(Pid, {async_set_socket_options, Options}). + +cast(Pid, Req) -> + gen_server:cast(Pid, Req). + call(Pid, Req) -> call(Pid, Req, infinity). call(Pid, Req, Timeout) -> @@ -366,6 +400,9 @@ handle_msg({'$gen_call', From, Req}, State) -> gen_server:reply(From, Reply), stop(Reason, NState) end; +handle_msg({'$gen_cast', Req}, State) -> + NewState = handle_cast(Req, State), + {ok, NewState}; handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> ?LOG(debug, "RECV ~0p", [Data]), @@ -692,6 +729,22 @@ handle_info({sock_error, Reason}, State) -> handle_info(Info, State) -> with_channel(handle_info, [Info], State). +%%-------------------------------------------------------------------- +%% Handle Info + +handle_cast({async_set_socket_options, Opts}, + State = #state{transport = Transport, + socket = Socket + }) -> + case Transport:setopts(Socket, Opts) of + ok -> ?tp(info, "custom_socket_options_successfully", #{opts => Opts}); + Err -> ?tp(error, "failed_to_set_custom_socket_optionn", #{reason => Err}) + end, + State; +handle_cast(Req, State) -> + ?tp(error, "received_unknown_cast", #{cast => Req}), + State. + %%-------------------------------------------------------------------- %% Ensure rate limit @@ -820,3 +873,7 @@ set_field(Name, Value, State) -> Pos = emqx_misc:index_of(Name, record_info(fields, state)), setelement(Pos+1, State, Value). +get_state(Pid) -> + State = sys:get_state(Pid), + maps:from_list(lists:zip(record_info(fields, state), + tl(tuple_to_list(State)))). diff --git a/test/emqx_mqtt_SUITE.erl b/test/emqx_mqtt_SUITE.erl index cb6174712..c86d6334a 100644 --- a/test/emqx_mqtt_SUITE.erl +++ b/test/emqx_mqtt_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(STATS_KYES, [recv_pkt, recv_msg, send_pkt, send_msg, recv_oct, recv_cnt, send_oct, send_cnt, @@ -38,6 +39,19 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). +init_per_testcase(TestCase, Config) -> + case erlang:function_exported(?MODULE, TestCase, 2) of + true -> ?MODULE:TestCase(init, Config); + false -> Config + end. + +end_per_testcase(TestCase, Config) -> + case erlang:function_exported(?MODULE, TestCase, 2) of + true -> ?MODULE:TestCase('end', Config); + false -> ok + end, + Config. + t_conn_stats(_) -> with_client(fun(CPid) -> Stats = emqx_connection:stats(CPid), @@ -134,3 +148,41 @@ with_client(TestFun, _Options) -> emqtt:stop(C) end. +t_async_set_keepalive(init, Config) -> + ok = snabbkaffe:start_trace(), + Config; +t_async_set_keepalive('end', _Config) -> + snabbkaffe:stop(), + ok. + +t_async_set_keepalive(_) -> + ClientID = <<"client-tcp-keepalive">>, + {ok, Client} = emqtt:start_link([{host, "localhost"}, + {proto_ver,v5}, + {clientid, ClientID}, + {clean_start, false}]), + {ok, _} = emqtt:connect(Client), + {ok, _} = ?block_until(#{?snk_kind := insert_channel_info, + client_id := ClientID}, 2000, 100), + [Pid] = emqx_cm:lookup_channels(ClientID), + State = emqx_connection:get_state(Pid), + Transport = maps:get(transport, State), + Socket = maps:get(socket, State), + ?assert(is_port(Socket)), + Opts = [{raw, 6, 4, 4}, {raw, 6, 5, 4}, {raw, 6, 6, 4}], + {ok, [ {raw, 6, 4, <>} + , {raw, 6, 5, <>} + , {raw, 6, 6, <>} + ]} = Transport:getopts(Socket, Opts), + ct:pal("Idle=~p, Interval=~p, Probes=~p", [Idle, Interval, Probes]), + emqx_connection:async_set_keepalive(Pid, Idle + 1, Interval + 1, Probes + 1), + {ok, _} = ?block_until(#{?snk_kind := "custom_socket_options_successfully"}, 1000), + {ok, [ {raw, 6, 4, <>} + , {raw, 6, 5, <>} + , {raw, 6, 6, <>} + ]} = Transport:getopts(Socket, Opts), + ?assertEqual(NewIdle, Idle + 1), + ?assertEqual(NewInterval, Interval + 1), + ?assertEqual(NewProbes, Probes + 1), + emqtt:stop(Client), + ok.