From 8d138d2525f5c34e5f4afb830f2b49c94a6bde77 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 20 Nov 2022 14:29:28 +0100 Subject: [PATCH 01/45] ci: build slim packages on macos arm64 --- .github/actions/package-macos/action.yaml | 29 ++++++++++++++-------- .github/workflows/build_slim_packages.yaml | 4 +-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 57c7910a8..efa9ac812 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -30,25 +30,29 @@ runs: shell: bash run: | brew update - brew install curl zip unzip kerl coreutils openssl@1.1 + brew install curl zip unzip coreutils openssl@1.1 echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH + echo "OTP_SOURCE_PATH=$HOME/otp-${{ inputs.otp }}" >> $GITHUB_ENV + echo "OTP_INSTALL_PATH=/opt/erlang/${{ inputs.otp }}" >> $GITHUB_ENV - uses: actions/cache@v3 id: cache with: - path: ~/.kerl/${{ inputs.otp }} + path: /opt/erlang/${{ inputs.otp }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit - name: build erlang if: steps.cache.outputs.cache-hit != 'true' shell: bash - env: - KERL_BUILD_BACKEND: git - OTP_GITHUB_URL: https://github.com/emqx/otp - KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit run: | - kerl update releases - kerl build ${{ inputs.otp }} - kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }} + if [ -d "$OTP_SOURCE_PATH" ]; then + rm -rf "$OTP_SOURCE_PATH" + fi + git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH" + cd "$OTP_SOURCE_PATH" + ./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH" + make -j$(nproc) + rm -rf "$OTP_INSTALL_PATH" + make install - name: build ${{ inputs.profile }} env: AUTO_INSTALL_BUILD_DEPS: 1 @@ -61,13 +65,16 @@ runs: APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }} shell: bash run: | - . $HOME/.kerl/${{ inputs.otp }}/activate + export PATH="$OTP_INSTALL_PATH/bin:$PATH" make ensure-rebar3 - sudo cp rebar3 /usr/local/bin/rebar3 + mkdir -p $HOME/bin + cp rebar3 $HOME/bin/rebar3 + export PATH="$HOME/bin:$PATH" make ${{ inputs.profile }}-tgz - name: test ${{ inputs.profile }} shell: bash run: | + export PATH="$OTP_INSTALL_PATH/bin:$PATH" pkg_name=$(find _packages/${{ inputs.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) mkdir emqx unzip -d emqx $pkg_name > /dev/null diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 0e3e3d036..1a5cd6c77 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -135,6 +135,7 @@ jobs: - 24.3.4.2-1 os: - macos-11 + - macos-12-arm64 runs-on: ${{ matrix.os }} @@ -143,7 +144,6 @@ jobs: - name: prepare run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - echo "BUILD_WITH_QUIC=1" >> $GITHUB_ENV - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} @@ -155,7 +155,7 @@ jobs: apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v3 with: - name: macos + name: ${{ matrix.os }} path: _packages/**/* spellcheck: From f02e4af09a7682bdda828bf04c0f881fe8fa067e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 20 Nov 2022 16:19:28 +0100 Subject: [PATCH 02/45] ci: use user home dir for otp install --- .github/actions/package-macos/action.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index efa9ac812..52581c4c4 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -31,14 +31,15 @@ runs: run: | brew update brew install curl zip unzip coreutils openssl@1.1 + mkdir -p $HOME/src $HOME/otp echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH - echo "OTP_SOURCE_PATH=$HOME/otp-${{ inputs.otp }}" >> $GITHUB_ENV - echo "OTP_INSTALL_PATH=/opt/erlang/${{ inputs.otp }}" >> $GITHUB_ENV + echo "OTP_SOURCE_PATH=$HOME/src/otp-${{ inputs.otp }}" >> $GITHUB_ENV + echo "OTP_INSTALL_PATH=$HOME/otp/${{ inputs.otp }}" >> $GITHUB_ENV - uses: actions/cache@v3 id: cache with: - path: /opt/erlang/${{ inputs.otp }} + path: $HOME/otp/${{ inputs.otp }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit - name: build erlang if: steps.cache.outputs.cache-hit != 'true' From 262e53eed258a011601e3e83090708f17450ad53 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 20 Nov 2022 16:38:55 +0100 Subject: [PATCH 03/45] chore: ensure macos keychain does not exist when signing binaries --- scripts/macos-sign-binaries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh index fed55faef..68a0216fa 100755 --- a/scripts/macos-sign-binaries.sh +++ b/scripts/macos-sign-binaries.sh @@ -24,6 +24,7 @@ base64 --decode > "${PKSC12_FILE}" <<<"${APPLE_DEVELOPER_ID_BUNDLE}" KEYCHAIN='emqx.keychain-db' KEYCHAIN_PASSWORD="$(openssl rand -base64 32)" +security delete-keychain "${KEYCHAIN}" 2>/dev/null || true security create-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}" security set-keychain-settings -lut 21600 "${KEYCHAIN}" security unlock-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}" From c21fa3c33912b12646507bb0477cf52fea23b7d8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 22 Nov 2022 09:51:30 +0800 Subject: [PATCH 04/45] fix(typo): typo in code --- apps/emqx/src/emqx_ws_connection.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index c7c31a2d8..9b3b34493 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -378,10 +378,14 @@ get_peer_info(Type, Listener, Req, Opts) -> of #{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} -> SourceName = {SrcAddr, SrcPort}, - %% Notice: Only CN is available in Proxy Protocol V2 additional info + %% Notice: CN is only available in Proxy Protocol V2 additional info. + %% `CN` is unsupported in Proxy Protocol V1 + %% `pp2_ssl_cn` is required by config `peer_cert_as_username` or `peer_cert_as_clientid`. + %% It will be parsed by esockd. + %% See also `emqx_channel:set_peercert_infos/3` and `esockd_peercert:common_name/1` SourceSSL = case maps:get(cn, SSL, undefined) of - undeined -> nossl; + undefined -> undefined; CN -> [{pp2_ssl_cn, CN}] end, {SourceName, SourceSSL}; From d0d99378b4f7413079c3decd393a2dd6b809cac0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 22 Nov 2022 11:01:12 +0100 Subject: [PATCH 05/45] ci: define otp source and install path in one place --- .github/actions/package-macos/action.yaml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 52581c4c4..37284e8b6 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -26,25 +26,30 @@ inputs: runs: using: composite steps: - - name: prepare + - id: prepare shell: bash run: | brew update brew install curl zip unzip coreutils openssl@1.1 - mkdir -p $HOME/src $HOME/otp echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH - echo "OTP_SOURCE_PATH=$HOME/src/otp-${{ inputs.otp }}" >> $GITHUB_ENV - echo "OTP_INSTALL_PATH=$HOME/otp/${{ inputs.otp }}" >> $GITHUB_ENV + echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT + OTP_SOURCE_PATH="$HOME/src/otp-${{ inputs.otp }}" + OTP_INSTALL_PATH="$HOME/otp/${{ inputs.otp }}" + echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT + echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT + mkdir -p "$OTP_SOURCE_PATH" "$OTP_INSTALL_PATH" - uses: actions/cache@v3 id: cache with: - path: $HOME/otp/${{ inputs.otp }} + path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit - name: build erlang if: steps.cache.outputs.cache-hit != 'true' shell: bash run: | + OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}" + OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}" if [ -d "$OTP_SOURCE_PATH" ]; then rm -rf "$OTP_SOURCE_PATH" fi @@ -66,7 +71,7 @@ runs: APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }} shell: bash run: | - export PATH="$OTP_INSTALL_PATH/bin:$PATH" + export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH" make ensure-rebar3 mkdir -p $HOME/bin cp rebar3 $HOME/bin/rebar3 @@ -75,7 +80,7 @@ runs: - name: test ${{ inputs.profile }} shell: bash run: | - export PATH="$OTP_INSTALL_PATH/bin:$PATH" + export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH" pkg_name=$(find _packages/${{ inputs.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) mkdir emqx unzip -d emqx $pkg_name > /dev/null From 7b5f3f53040388b5c77a3b15235b1dcc66da6c27 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Nov 2022 15:26:48 +0100 Subject: [PATCH 06/45] docs: add change log files for v5.0.12 --- changes/v5.0.12-en.md | 5 +++++ changes/v5.0.12-zh.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 changes/v5.0.12-en.md create mode 100644 changes/v5.0.12-zh.md diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md new file mode 100644 index 000000000..52af23519 --- /dev/null +++ b/changes/v5.0.12-en.md @@ -0,0 +1,5 @@ +# v5.0.12 + +## Enhancements + +## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md new file mode 100644 index 000000000..49f479a1e --- /dev/null +++ b/changes/v5.0.12-zh.md @@ -0,0 +1,5 @@ +# v5.0.12 + +## 增强 + +## 修复 From 282e6ff218e18e82d8706b0bb01f70eba4e34473 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 24 Nov 2022 09:59:39 +0800 Subject: [PATCH 07/45] chore: set channel info once --- apps/emqx/src/emqx_connection.erl | 3 +-- apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 4ebc5b5e6..d892f5acc 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -601,8 +601,7 @@ handle_msg({event, disconnected}, State = #state{channel = Channel}) -> {ok, State}; handle_msg({event, _Other}, State = #state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), - emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:set_chan_stats(ClientId, stats(State)), + emqx_cm:insert_channel_info(ClientId, info(State), stats(State)), {ok, State}; handle_msg({timeout, TRef, TMsg}, State) -> handle_timeout(TRef, TMsg, State); diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl index cb4c61771..df66e2854 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl @@ -16,7 +16,7 @@ -module(emqx_gateway_impl). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -type state() :: map(). -type reason() :: any(). From 03fb543ba83013888adfc7500bed7e04636cc7ff Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 24 Nov 2022 17:28:53 +0800 Subject: [PATCH 08/45] fix(ctl): fix unsafe `list_to_atom` A command can be called only when it is exists --- apps/emqx/src/emqx_ctl.erl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl index 8e8477f7c..1087d9282 100644 --- a/apps/emqx/src/emqx_ctl.erl +++ b/apps/emqx/src/emqx_ctl.erl @@ -103,7 +103,13 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). run_command([]) -> run_command(help, []); run_command([Cmd | Args]) -> - run_command(list_to_atom(Cmd), Args). + case emqx_misc:safe_to_existing_atom(Cmd) of + {ok, Cmd1} -> + run_command(Cmd1, Args); + _ -> + help(), + {error, cmd_not_found} + end. -spec run_command(cmd(), list(string())) -> ok | {error, term()}. run_command(help, []) -> @@ -220,12 +226,13 @@ init([]) -> handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) -> case ets:match(?CMD_TAB, {{'$1', Cmd}, '_', '_'}) of [] -> - ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}); + ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}), + {reply, ok, next_seq(State)}; [[OriginSeq] | _] -> ?SLOG(warning, #{msg => "CMD_overidden", cmd => Cmd, mf => MF}), - true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}) - end, - {reply, ok, next_seq(State)}; + true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}), + {reply, ok, State} + end; handle_call(Req, _From, State) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. From 723959fe0df498d67cb568318f5d975fae49052a Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 24 Nov 2022 17:49:35 +0800 Subject: [PATCH 09/45] chore: update changes --- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 52af23519..e3217a89a 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -2,4 +2,6 @@ ## Enhancements +- Improve the CLI to avoid waste atom table when typing erros [#9416](https://github.com/emqx/emqx/pull/9416). + ## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 49f479a1e..0254619c4 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -3,3 +3,6 @@ ## 增强 ## 修复 + +- 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 + From 70b6397adae858365a5e6d911d539b05a4537997 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 24 Nov 2022 20:10:37 +0800 Subject: [PATCH 10/45] feat: support node.global_gc_interval=disabled --- apps/emqx_conf/i18n/emqx_conf_schema.conf | 4 ++-- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- apps/emqx_conf/test/emqx_global_gc_SUITE.erl | 6 +++++- apps/emqx_machine/src/emqx_global_gc.erl | 6 +++--- changes/v5.0.11-en.md | 2 ++ changes/v5.0.11-zh.md | 2 ++ 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index fc33589ec..6123b313f 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -471,8 +471,8 @@ significant: later configuration files override the previous ones. node_global_gc_interval { desc { - en: """Periodic garbage collection interval.""" - zh: """系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。""" + en: """Periodic garbage collection interval. Set to disabled if not enabled.""" + zh: """系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" } label { en: "Global GC Interval" diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a68c11dc4..2080d32cb 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -468,7 +468,7 @@ fields("node") -> )}, {"global_gc_interval", sc( - emqx_schema:duration(), + hoconsc:union([disabled, emqx_schema:duration()]), #{ mapping => "emqx_machine.global_gc_interval", default => "15m", diff --git a/apps/emqx_conf/test/emqx_global_gc_SUITE.erl b/apps/emqx_conf/test/emqx_global_gc_SUITE.erl index 121e12eba..9e4a5ffa6 100644 --- a/apps/emqx_conf/test/emqx_global_gc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_global_gc_SUITE.erl @@ -29,4 +29,8 @@ t_run_gc(_) -> ok = timer:sleep(1500), {ok, MilliSecs} = emqx_global_gc:run(), ct:print("Global GC: ~w(ms)~n", [MilliSecs]), - emqx_global_gc:stop(). + emqx_global_gc:stop(), + ok = emqx_config:put([node, global_gc_interval], disabled), + {ok, Pid} = emqx_global_gc:start_link(), + ?assertMatch(#{timer := undefined}, sys:get_state(Pid)), + ok. diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index 74825c321..a704d603c 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -82,10 +82,10 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- ensure_timer(State) -> - case application:get_env(emqx_machine, global_gc_interval) of - undefined -> + case application:get_env(emqx_machine, global_gc_interval, disabled) of + disabled -> State; - {ok, Interval} -> + Interval when is_integer(Interval) -> TRef = emqx_misc:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md index e53c5785e..ee00370fd 100644 --- a/changes/v5.0.11-en.md +++ b/changes/v5.0.11-en.md @@ -23,6 +23,8 @@ - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398). +- Disable global garbage collection by `node.global_gc_interval = disabled` [#9418](https://github.com/emqx/emqx/pull/9418)。 + ## Bug fixes - Fix `ssl.existingName` option of helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307). diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md index 3ea516dad..328c17467 100644 --- a/changes/v5.0.11-zh.md +++ b/changes/v5.0.11-zh.md @@ -21,6 +21,8 @@ - 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。 +- 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 + ## 修复 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。 From 59ac5320b1da00dfdc62521b5b14a299d3cb6ef3 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 24 Nov 2022 20:38:11 +0800 Subject: [PATCH 11/45] chore: update emqx_conf/emqx_machine version --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 2b6efe639..c57664ca7 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 59a5ad4b8..63c6c01ad 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib]}, From e4ec72ae9eff2c329e43838e17ca49b1ae73feb5 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 25 Nov 2022 07:33:36 +0800 Subject: [PATCH 12/45] chore: update v5.0.12 changelog --- changes/v5.0.11-en.md | 2 -- changes/v5.0.11-zh.md | 2 -- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 2 ++ 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md index ee00370fd..e53c5785e 100644 --- a/changes/v5.0.11-en.md +++ b/changes/v5.0.11-en.md @@ -23,8 +23,6 @@ - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398). -- Disable global garbage collection by `node.global_gc_interval = disabled` [#9418](https://github.com/emqx/emqx/pull/9418)。 - ## Bug fixes - Fix `ssl.existingName` option of helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307). diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md index 328c17467..3ea516dad 100644 --- a/changes/v5.0.11-zh.md +++ b/changes/v5.0.11-zh.md @@ -21,8 +21,6 @@ - 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。 -- 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 - ## 修复 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。 diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 52af23519..f61778840 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -2,4 +2,6 @@ ## Enhancements +- Disable global garbage collection by `node.global_gc_interval = disabled` [#9418](https://github.com/emqx/emqx/pull/9418)。 + ## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 49f479a1e..41507fe23 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -2,4 +2,6 @@ ## 增强 +- 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 + ## 修复 From 28bede0377427e68b864cc63023efacdd288e372 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 25 Nov 2022 16:38:40 +0800 Subject: [PATCH 13/45] chore: update i18n en description --- apps/emqx_conf/i18n/emqx_conf_schema.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 6123b313f..337823233 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -471,7 +471,7 @@ significant: later configuration files override the previous ones. node_global_gc_interval { desc { - en: """Periodic garbage collection interval. Set to disabled if not enabled.""" + en: """Periodic garbage collection interval. Set to disabled to have it disabled.""" zh: """系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" } label { From 5c40b1dffd5eb3a374d3243b54962db7041d0c02 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 25 Nov 2022 13:09:02 +0100 Subject: [PATCH 14/45] ci: build macos release packages for apple silicon (arm64) --- .github/workflows/build_packages.yaml | 9 ++++----- changes/v5.0.11-en.md | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b738ab999..5149b5d46 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -74,7 +74,7 @@ jobs: esac ;; esac - echo "::set-output name=BUILD_PROFILE::$PROFILE" + echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT - name: get_all_deps run: | make -C source deps-all @@ -143,8 +143,10 @@ jobs: - 24.3.4.2-1 os: - macos-11 + - macos-12-arm64 runs-on: ${{ matrix.os }} steps: + - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/download-artifact@v3 with: name: source @@ -152,11 +154,8 @@ jobs: - name: unzip source code run: | ln -s . source - unzip -q source.zip + unzip -o -q source.zip rm source source.zip - - name: prepare - run: | - git config --global credential.helper store - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md index e53c5785e..3bf1c6aa7 100644 --- a/changes/v5.0.11-en.md +++ b/changes/v5.0.11-en.md @@ -23,6 +23,8 @@ - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398). +- Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). + ## Bug fixes - Fix `ssl.existingName` option of helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307). From a14d40e596a6ab9e29ea74c0848991c8a1575f5e Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 25 Nov 2022 21:39:12 +0800 Subject: [PATCH 15/45] chore: update changes --- changes/v5.0.11-en.md | 2 -- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md index 3bf1c6aa7..e53c5785e 100644 --- a/changes/v5.0.11-en.md +++ b/changes/v5.0.11-en.md @@ -23,8 +23,6 @@ - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398). -- Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). - ## Bug fixes - Fix `ssl.existingName` option of helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307). diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index e3217a89a..5f72cb174 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -4,4 +4,6 @@ - Improve the CLI to avoid waste atom table when typing erros [#9416](https://github.com/emqx/emqx/pull/9416). +- Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). + ## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 0254619c4..ea7c8793d 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -6,3 +6,4 @@ - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 +- 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 From 5af1d6662f3583e96d9cdb2ff1c1fdf187f18239 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Tue, 22 Nov 2022 16:22:36 +0300 Subject: [PATCH 16/45] chore: update grafana dashboards --- .../emqx_prometheus/grafana_template/EMQ.json | 2690 +++++++++-------- .../grafana_template/EMQ_Dashboard.json | 917 +++--- .../grafana_template/ErlangVM.json | 1957 ++++++------ 3 files changed, 2993 insertions(+), 2571 deletions(-) diff --git a/apps/emqx_prometheus/grafana_template/EMQ.json b/apps/emqx_prometheus/grafana_template/EMQ.json index 54d0a9d47..28b005026 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ.json +++ b/apps/emqx_prometheus/grafana_template/EMQ.json @@ -4,531 +4,626 @@ { "$$hashKey": "object:13", "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, - "iteration": 1620639954651, - "links": [], + "id": 35, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "emqx" + ], + "targetBlank": false, + "title": "EMQX dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, - "height": "", - "hiddenSeries": false, "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "emqx_connections_count{instance=\"$host\"}", + "expr": "emqx_connections_count{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "interval": "1m", "intervalFactor": 1, "legendFormat": "Current", + "range": true, "refId": "A", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_connections_max{instance=\"$host\"}", + "expr": "emqx_connections_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "1m", "legendFormat": "Max", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Clients", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, - "hiddenSeries": false, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_sessions_count{instance=\"$host\"}", + "expr": "emqx_sessions_count{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "Current", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_sessions_max{instance=\"$host\"}", + "expr": "emqx_sessions_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Max", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Session", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Sessions", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, - "hiddenSeries": false, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_suboptions_count{instance=\"$host\"}", + "expr": "emqx_suboptions_count{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "Subopt-Current", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_suboptions_max{instance=\"$host\"}", + "expr": "emqx_suboptions_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Subopt-Max", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_subscribers_count{instance=\"$host\"}", + "expr": "emqx_subscribers_count{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Suber-Current", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_subscribers_max{instance=\"$host\"}", + "expr": "emqx_subscribers_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Suber-Max", "refId": "D" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Subscribers & Suboptions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Subscribers and suboptions", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, - "hiddenSeries": false, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "emqx_subscriptions_count{instance=\"$host\"}", + "expr": "emqx_subscriptions_count{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Normal-Current", + "range": true, "refId": "A", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_subscriptions_max{instance=\"$host\"}", + "expr": "emqx_subscriptions_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Normal-Max", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_subscriptions_shared_count{instance=\"$host\"}", + "expr": "emqx_subscriptions_shared_count{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Shared-Current", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_subscriptions_shared_max{instance=\"$host\"}", + "expr": "emqx_subscriptions_shared_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Shared-Max", "refId": "D" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Subscriptions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 }, - "hiddenSeries": false, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_routes_count{instance=\"$host\"}", + "expr": "emqx_routes_count{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -536,921 +631,1005 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_routes_max{instance=\"$host\"}", + "expr": "emqx_routes_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Max", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Topics", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 }, - "hiddenSeries": false, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_retained_count{instance=\"$host\"}", + "expr": "emqx_retained_count{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "Current", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_retained_max{instance=\"$host\"}", + "expr": "emqx_retained_max{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Max", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Retained Messages", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Retained messages", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 20 }, - "hiddenSeries": false, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Received", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Sent", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_retained{instance=\"$host\"}[$__interval]) or irate(emqx_messages_retained{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_retained{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_retained{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Retained", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_dropped{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_dropped{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Dropped", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_delayed{instance=\"$host\"}[$__interval]) or irate(emqx_messages_delayed{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_delayed{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_delayed{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delayed", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_forward{instance=\"$host\"}[$__interval]) or irate(emqx_messages_forward{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_forward{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_forward{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Forward", "refId": "F" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages Overview", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Messages overview", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 20 }, - "hiddenSeries": false, "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_dropped{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_dropped{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Message.Dropped", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_dropped_expired {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_dropped_expired {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Message.Dropped.Expired", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_dropped_no_subscribers {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_dropped_no_subscribers {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Message.Dropped.no_subscribers", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped_no_local{instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_no_local {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped_no_local{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped_no_local {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped.no_local", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped_too_large {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped_too_large {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped.too_large", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped_qos0_msg {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped_qos0_msg {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped.qos0_msg", "refId": "G" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped_queue_full {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped_queue_full {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped.queue_full", "refId": "H" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_delivery_dropped_expired {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_delivery_dropped_expired {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Delivery.Dropped.expired", "refId": "I" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages/Delivery Dropped", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Messages/delivery dropped", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 26 }, - "hiddenSeries": false, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Received", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": false, - "expr": "sum(rate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos0_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos0_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Received.QoS0", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos1_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos1_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Received.QoS1", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos2_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos2_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Received.QoS2", "refId": "D" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages Received", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Messages received", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 26 }, - "hiddenSeries": false, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(emqx_messages_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Sent", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos0_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos0_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Sent.QoS0", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos1_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos1_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Sent.QoS1", "refId": "G" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_qos2_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_qos2_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Sent.QoS2", "refId": "H" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Messages Sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Messages sent", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 32 }, - "hiddenSeries": false, "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "sum(rate(emqx_bytes_received{instance=\"$host\"}[$__interval]) or irate(emqx_bytes_received{instance=\"$host\"}[$__interval]))", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(emqx_bytes_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_bytes_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Bytes received", "refId": "A", "step": 60 }, { - "expr": "sum(rate(emqx_bytes_sent{instance=\"$host\"}[$__interval]) or irate(emqx_bytes_sent{instance=\"$host\"}[$__interval]))", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(emqx_bytes_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_bytes_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Bytes sent", "refId": "B", "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 32 }, - "hiddenSeries": false, "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_connect{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_connect{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", "hide": false, - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Connect", "refId": "A", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_connack_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Connack sent", "refId": "B", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_disconnect_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Disconnect sent", "refId": "C", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_connack_error {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_connack_error {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "connack_error", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_connack_auth_error {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_connack_auth_error {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "intervalFactor": 1, @@ -1458,124 +1637,126 @@ "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_auth_received {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_auth_received {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "Auth", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_auth_sent {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_auth_sent {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "Auth sent", "refId": "G" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Packets connect", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 38 }, - "hiddenSeries": false, "hideTimeOverride": false, "id": 1, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_subscribe_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", "instant": false, - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Subscribe received", "metric": "", @@ -1583,153 +1764,173 @@ "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_suback_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Suback sent", "refId": "B", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_unsubscribe_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Unsubscribe received", "refId": "C", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_unsuback_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_unsuback_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Unsuback sent", "refId": "D", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_subscribe_auth_error {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_subscribe_auth_error {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "sub_auth_error", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_subscribe_error {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_subscribe_error {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "sub_error", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_unsubscribe_error {instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_unsubscribe_error {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "unsub_error", "refId": "G" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Packets subscribe/unsubscribe", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 38 }, - "hiddenSeries": false, "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "sum(rate(emqx_packets_pingreq_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_pingreq_received{instance=\"$host\"}[$__interval]))", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(emqx_packets_pingreq_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_packets_pingreq_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1738,7 +1939,11 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_pingresp_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_pingresp_sent{instance=\"$host\"}[$__interval]))", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(emqx_packets_pingresp_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_packets_pingresp_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1747,319 +1952,275 @@ "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Packets ping", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 44 }, - "hiddenSeries": false, "hideTimeOverride": false, "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_publish_received{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_publish_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Publish received", "refId": "A", "step": 15 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_publish_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Publish sent", "refId": "B", "step": 15 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "rate(emqx_packets_puback_received{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_puback_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 1, "legendFormat": "Puback received", "metric": "", + "range": true, "refId": "C", "step": 15 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "rate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval])", + "expr": "rate(emqx_packets_puback_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Puback sent", + "range": true, "refId": "D", "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_pubrel_received{instance=\"$host\"}[$interval])", + "expr": "rate(emqx_packets_pubrel_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Pubrel received", "refId": "E", "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_pubrel_sent{instance=\"$host\"}[$interval])", + "expr": "rate(emqx_packets_pubrel_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Pubrel sent", "refId": "F", "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_pubcomp_received{instance=\"$host\"}[$interval])", + "expr": "rate(emqx_packets_pubcomp_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Pubcomp received", "refId": "G", "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$interval])", + "expr": "rate(emqx_packets_pubcomp_sent{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "format": "time_series", - "interval": "$interval", "intervalFactor": 2, "legendFormat": "Pubcomp sent", "refId": "H", "step": 30 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Packets publish", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" } ], "refresh": false, - "schemaVersion": 27, + "schemaVersion": 37, "style": "dark", - "tags": [], + "tags": [ + "emqx" + ], "templating": { "list": [ { - "auto": true, - "auto_count": 200, - "auto_min": "1s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "description": null, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1s", - "value": "1s" - }, - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "1s,5s,1m,5m,1h,6h,1d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "128.160.171.92:8081", - "value": "128.160.171.92:8081" - }, - "datasource": "Prometheus", - "definition": "label_values(emqx_connections_count, instance)", - "description": null, - "error": null, "hide": 0, "includeAll": false, - "label": "Host", + "label": "", "multi": false, - "name": "host", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(emqx_connections_count, job)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "job", "options": [], "query": { - "query": "label_values(emqx_connections_count, instance)", - "refId": "Prometheus-host-Variable-Query" + "query": "label_values(emqx_connections_count, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(emqx_connections_count{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(emqx_connections_count{job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -2067,7 +2228,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-12h", "to": "now" }, "timepicker": { @@ -2096,7 +2257,8 @@ ] }, "timezone": "browser", - "title": "EMQ", + "title": "EMQ details", "uid": "tjRlQw6Zk", - "version": 29 -} + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json index 0ee3c6741..319e539ce 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json +++ b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json @@ -4,84 +4,156 @@ { "$$hashKey": "object:191", "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, - "links": [], + "id": 33, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "emqx" + ], + "targetBlank": false, + "title": "EMQX dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 0 }, - "height": "", - "hiddenSeries": false, "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(emqx_connections_count{instance=~\".*\",job=\"emqx\"})", + "expr": "sum(emqx_connections_count{instance=~\"$instance\",job=~\"$job\"})", "hide": false, "interval": "", "legendFormat": "Clients", + "range": true, "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_client_connected{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_client_connected{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "instant": false, "interval": "1m", @@ -91,8 +163,13 @@ "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_client_disconnected{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_client_disconnected{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "instant": false, "interval": "1m", @@ -102,503 +179,554 @@ "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Clients", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:69", - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:70", - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 }, - "hiddenSeries": false, "id": 11, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_messages_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_received{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, "legendFormat": "received", + "range": true, "refId": "A", "step": 120 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_messages_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_messages_sent{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, "legendFormat": "sent", + "range": true, "refId": "B", "step": 120 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Messages", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 18 }, - "hiddenSeries": false, "id": 13, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "emqx_routes_count{instance=~\".*\",job=\"emqx\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "emqx_routes_count{instance=~\"$instance\",job=~\"$job\"}\nor emqx_topics_count{instance=~\"$instance\",job=~\"$job\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Topics", + "range": true, "refId": "A", "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Topics", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:236", - "decimals": -2, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:237", - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 25 }, - "hiddenSeries": false, "id": 8, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "emqx_subscriptions_count{instance=~\".*\",job=\"emqx\"}", + "expr": "emqx_subscriptions_count{instance=~\"$instance\",job=~\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Subscriptions", + "range": true, "refId": "A", "step": 60 }, { - "expr": "emqx_subscriptions_shared_count{instance=~\".*\",job=\"emqx\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "emqx_subscriptions_shared_count{instance=~\"$instance\",job=~\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Shared Subscriptions", + "range": true, "refId": "B", "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Subscriptions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:157", - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:158", - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 32 }, - "hiddenSeries": false, "id": 9, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_bytes_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_bytes_received{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Bytes received", + "range": true, "refId": "A", "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum(rate(emqx_bytes_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", + "expr": "sum(rate(emqx_bytes_sent{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Bytes sent", + "range": true, "refId": "B", "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" } ], "refresh": false, - "schemaVersion": 27, + "schemaVersion": 37, "style": "dark", - "tags": [], + "tags": [ + "emqx" + ], "templating": { - "list": [] + "list": [ + { + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(emqx_cluster_nodes_running, job)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(emqx_cluster_nodes_running, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(emqx_cluster_nodes_running{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(emqx_cluster_nodes_running{job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { - "from": "now-6h", + "from": "now-12h", "to": "now" }, "timepicker": { @@ -627,7 +755,8 @@ ] }, "timezone": "browser", - "title": "EMQ Dashboard", + "title": "EMQ overview", "uid": "5sreUw6Wz", - "version": 11 -} + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/ErlangVM.json b/apps/emqx_prometheus/grafana_template/ErlangVM.json index 23088123c..ca4d37bc3 100644 --- a/apps/emqx_prometheus/grafana_template/ErlangVM.json +++ b/apps/emqx_prometheus/grafana_template/ErlangVM.json @@ -4,472 +4,542 @@ { "$$hashKey": "object:420", "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, - "iteration": 1620639927619, - "links": [], + "id": 36, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "emqx" + ], + "targetBlank": false, + "title": "EMQX dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The CPU usage for the hosted machine", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "hiddenSeries": false, "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "emqx_vm_cpu_use {instance=\"$host\"}", + "expr": "emqx_vm_cpu_use{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "used", + "range": true, "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_vm_cpu_idle {instance=\"$host\"}", + "expr": "emqx_vm_cpu_idle {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "idle", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The memory usage for the hosted machine", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "hiddenSeries": false, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_vm_total_memory{instance=\"$host\"}", + "expr": "emqx_vm_total_memory{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "total", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_vm_used_memory{instance=\"$host\"}", + "expr": "emqx_vm_used_memory{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "used", "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The total amount of memory currently allocated.\nThis is the same as the sum of the memory size for processes and system.", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 8 }, - "hiddenSeries": false, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_memory_bytes_total {instance=\"$host\"}", + "expr": "erlang_vm_memory_bytes_total{job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "{{kind}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "VM Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "VM memory", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The total amount of memory currently allocated for the Erlang processes.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 8 }, - "hiddenSeries": false, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "erlang_vm_memory_processes_bytes_total{instance=\"$host\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "erlang_vm_memory_processes_bytes_total{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{usage}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Memory processes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "The total amount of memory currently allocated for the emulator that is not directly related to any Erlang process.\nMemory presented as processes is not included in this memory.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 15 }, - "hiddenSeries": false, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "erlang_vm_memory_system_bytes_total{instance=\"$host\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "erlang_vm_memory_system_bytes_total{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{usage}}", @@ -477,557 +547,655 @@ "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Memory system", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 15 }, - "hiddenSeries": false, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_logical_processors {instance=\"$host\"}", + "expr": "erlang_vm_logical_processors {job=~\"$job\", instance=\"$instance\"}", "interval": "", "legendFormat": "logical-processors", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_logical_processors_available {instance=\"$host\"}", + "expr": "erlang_vm_logical_processors_available {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "logical-processors-avai", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_logical_processors_online {instance=\"$host\"}", + "expr": "erlang_vm_logical_processors_online {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "logical-processors-online", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_schedulers {instance=\"$host\"}", + "expr": "erlang_vm_schedulers {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "schedulers", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_schedulers_online {instance=\"$host\"}", + "expr": "erlang_vm_schedulers_online {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "schedulers-online", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_threads {instance=\"$host\"}", + "expr": "erlang_vm_threads {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "vm_threads", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_thread_pool_size {instance=\"$host\"}", + "expr": "erlang_vm_thread_pool_size {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "vm_thread_pool_size", "refId": "G" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_vm_run_queue {instance=\"$host\"}", + "expr": "emqx_vm_run_queue {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "vm_run_queue", "refId": "H" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Processor & Scheduler & Thread", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Processor scheduler and threads", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, - "hiddenSeries": false, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_ets_limit {instance=\"$host\"}", + "expr": "erlang_vm_ets_limit {job=~\"$job\", instance=\"$instance\"}", "interval": "", - "legendFormat": "ETS Max", + "legendFormat": "ETS max", + "range": true, "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_ets_tables {instance=\"$host\"}", + "expr": "erlang_vm_ets_tables {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "ETS Count", + "legendFormat": "ETS count", + "range": true, "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_dets_tables {instance=\"$host\"}", + "expr": "erlang_vm_dets_tables {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "DETS Count", + "legendFormat": "DETS count", + "range": true, "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_process_limit {instance=\"$host\"}", + "expr": "erlang_vm_process_limit {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "Process Max", + "legendFormat": "Process max", + "range": true, "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_process_count {instance=\"$host\"}", + "expr": "erlang_vm_process_count {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "Process Count", + "legendFormat": "Process count", + "range": true, "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_port_limit {instance=\"$host\"}", + "expr": "erlang_vm_port_limit {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "Port Max", + "legendFormat": "Port max", + "range": true, "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", "exemplar": true, - "expr": "erlang_vm_port_count {instance=\"$host\"}", + "expr": "erlang_vm_port_count {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", - "legendFormat": "Port Count", + "legendFormat": "Port count", + "range": true, "refId": "G" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_smp_support {instance=\"$host\"}", + "expr": "erlang_vm_smp_support {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "smp_support", "refId": "H" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_vm_time_correction {instance=\"$host\"}", + "expr": "erlang_vm_time_correction {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "time_correction", "refId": "I" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "System Limit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "System limit", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, - "hiddenSeries": false, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_context_switches{instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_context_switches{job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "context_switches", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_garbage_collection_number_of_gcs {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_garbage_collection_number_of_gcs {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "garbage_collection_number_of_gcs", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_garbage_collection_bytes_reclaimed {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_garbage_collection_bytes_reclaimed {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "garbage_collection_bytes_reclaimed", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_bytes_received_total {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_bytes_received_total {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "bytes_received_total", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_bytes_output_total {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_bytes_output_total {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "bytes_output_total", "refId": "E" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_reductions_total {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_reductions_total {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "reductions", "refId": "F" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_runtime_milliseconds {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_runtime_milliseconds {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "runtime_milliseconds", "refId": "G" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(erlang_vm_statistics_wallclock_time_milliseconds {instance=\"$host\"}[$__interval])", + "expr": "rate(erlang_vm_statistics_wallclock_time_milliseconds {job=~\"$job\", instance=\"$instance\"}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "wallclock_time_milliseconds", "refId": "H" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "VM Statistics", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "VM statistics", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 29 }, - "hiddenSeries": false, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "erlang_vm_process_count{instance=\"$host\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "erlang_vm_process_count{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Process count", @@ -1035,7 +1203,11 @@ "step": 60 }, { - "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$host\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "erlang_vm_statistics_run_queues_length_total{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Run queue length", @@ -1043,105 +1215,109 @@ "step": 60 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "emqx_vm_process_messages_in_queues {instance=\"$host\"}", + "expr": "emqx_vm_process_messages_in_queues {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "messages_in_queue", "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Process", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 29 }, - "hiddenSeries": false, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { - "expr": "erlang_mnesia_lock_queue{instance=\"$host\"}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "erlang_mnesia_lock_queue{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Lock queue", @@ -1149,8 +1325,12 @@ "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_mnesia_held_locks{instance=\"$host\"}", + "expr": "erlang_mnesia_held_locks{job=~\"$job\", instance=\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1159,279 +1339,229 @@ "step": 30 }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_mnesia_transaction_coordinators{instance=\"$host\"}", + "expr": "erlang_mnesia_transaction_coordinators{job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Transaction coordinators", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "erlang_mnesia_transaction_participants {instance=\"$host\"}", + "expr": "erlang_mnesia_transaction_participants {job=~\"$job\", instance=\"$instance\"}", "hide": false, "interval": "", "legendFormat": "Transaction participants", "refId": "D" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Mnesia", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 }, - "hiddenSeries": false, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "7.5.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(erlang_mnesia_committed_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(erlang_mnesia_committed_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "Committed", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(erlang_mnesia_failed_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(erlang_mnesia_failed_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Failed", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(erlang_mnesia_restarted_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(erlang_mnesia_restarted_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Restart", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "sum(rate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]))", + "expr": "sum(rate(erlang_mnesia_logged_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(erlang_mnesia_logged_transactions {job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "Logged", "refId": "D" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Mnesia transaction", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" } ], "refresh": false, - "schemaVersion": 27, + "schemaVersion": 37, "style": "dark", - "tags": [], + "tags": [ + "emqx", + "erlang" + ], "templating": { "list": [ { - "auto": true, - "auto_count": 200, - "auto_min": "1s", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "description": null, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1s", - "value": "1s" - }, - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "1s,5s,1m,5m,1h,6h,1d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "128.160.171.92:8081", - "value": "128.160.171.92:8081" - }, - "datasource": "Prometheus", - "definition": "", - "description": null, - "error": null, "hide": 0, "includeAll": false, - "label": "Host", "multi": false, - "name": "host", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(erlang_vm_process_count, job)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "job", "options": [], "query": { - "query": "label_values(erlang_vm_process_count, instance)", - "refId": "Prometheus-host-Variable-Query" + "query": "label_values(erlang_vm_process_count, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(erlang_vm_process_count{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(erlang_vm_process_count{job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "$host.*", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -1439,7 +1569,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-30m", "to": "now" }, "timepicker": { @@ -1470,5 +1600,6 @@ "timezone": "browser", "title": "ErlangVM", "uid": "stprQQ6Zk", - "version": 13 -} + "version": 5, + "weekStart": "" +} \ No newline at end of file From 5c9fff43a269f9b7773c5d724e327b6920383025 Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Tue, 22 Nov 2022 16:58:54 +0300 Subject: [PATCH 17/45] chore: add proper units for rates on the dashboard --- .../emqx_prometheus/grafana_template/EMQ.json | 29 ++++++++----------- .../grafana_template/EMQ_Dashboard.json | 2 +- .../grafana_template/ErlangVM.json | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/apps/emqx_prometheus/grafana_template/EMQ.json b/apps/emqx_prometheus/grafana_template/EMQ.json index 28b005026..43af53a74 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ.json +++ b/apps/emqx_prometheus/grafana_template/EMQ.json @@ -793,7 +793,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -810,7 +809,7 @@ } ] }, - "unit": "short" + "unit": "mps" }, "overrides": [] }, @@ -841,11 +840,13 @@ "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "sum(rate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]) or irate(emqx_messages_received{job=~\"$job\", instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Received", + "range": true, "refId": "A" }, { @@ -953,7 +954,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -970,7 +970,7 @@ } ] }, - "unit": "short" + "unit": "mps" }, "overrides": [] }, @@ -1149,7 +1149,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -1166,7 +1165,7 @@ } ] }, - "unit": "short" + "unit": "mps" }, "overrides": [] }, @@ -1434,7 +1433,7 @@ } ] }, - "unit": "decbytes" + "unit": "Bps" }, "overrides": [] }, @@ -1528,7 +1527,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -1545,7 +1543,7 @@ } ] }, - "unit": "none" + "unit": "pps" }, "overrides": [] }, @@ -1704,7 +1702,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -1721,7 +1718,7 @@ } ] }, - "unit": "none" + "unit": "pps" }, "overrides": [] }, @@ -1882,7 +1879,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -1899,7 +1895,7 @@ } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [] }, @@ -1995,7 +1991,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -2012,7 +2007,7 @@ } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [] }, @@ -2259,6 +2254,6 @@ "timezone": "browser", "title": "EMQ details", "uid": "tjRlQw6Zk", - "version": 11, + "version": 12, "weekStart": "" -} \ No newline at end of file +} diff --git a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json index 319e539ce..324fbabd7 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json +++ b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json @@ -759,4 +759,4 @@ "uid": "5sreUw6Wz", "version": 9, "weekStart": "" -} \ No newline at end of file +} diff --git a/apps/emqx_prometheus/grafana_template/ErlangVM.json b/apps/emqx_prometheus/grafana_template/ErlangVM.json index ca4d37bc3..6e443d3fc 100644 --- a/apps/emqx_prometheus/grafana_template/ErlangVM.json +++ b/apps/emqx_prometheus/grafana_template/ErlangVM.json @@ -1602,4 +1602,4 @@ "uid": "stprQQ6Zk", "version": 5, "weekStart": "" -} \ No newline at end of file +} From 729cd8fda4864c8054e97f91361a8eb649d41db1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Nov 2022 22:42:52 +0100 Subject: [PATCH 18/45] build: add release helper scripts --- scripts/rel/check-chart-vsn.sh | 35 ++++++ scripts/rel/cut.sh | 214 +++++++++++++++++++++++++++++++++ scripts/rel/sync-remotes.sh | 156 ++++++++++++++++++++++++ 3 files changed, 405 insertions(+) create mode 100755 scripts/rel/check-chart-vsn.sh create mode 100755 scripts/rel/cut.sh create mode 100755 scripts/rel/sync-remotes.sh diff --git a/scripts/rel/check-chart-vsn.sh b/scripts/rel/check-chart-vsn.sh new file mode 100755 index 000000000..81ba32413 --- /dev/null +++ b/scripts/rel/check-chart-vsn.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "${DEBUG:-}" = 1 ]; then + set -x +fi + +# ensure dir +cd -P -- "$(dirname -- "$0")/../.." + +PROFILE="$1" +CHART_FILE="deploy/charts/${PROFILE}/Chart.yaml" + +if [ ! -f "$CHART_FILE" ]; then + echo "Chart file $CHART_FILE is not found" + echo "Current working dir: $(pwd)" + exit 1 +fi + +CHART_VSN="$(grep -oE '^version:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')" +APP_VSN="$(grep -oE '^appVersion:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')" + +if [ "$CHART_VSN" != "$APP_VSN" ]; then + echo "Chart version and app version mismatch in $CHART_FILE" + exit 2 +fi + +PKG_VSN="$(./pkg-vsn.sh "$PROFILE" | cut -d '-' -f 1)" + +if [ "$CHART_VSN" != "$PKG_VSN" ]; then + echo "Chart version in $CHART_FILE is not in sync with release version." + echo "Chart version: $CHART_VSN" + echo "Release version: $PKG_VSN" + exit 3 +fi diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh new file mode 100755 index 000000000..d28955bca --- /dev/null +++ b/scripts/rel/cut.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash + +## cut a new 5.x release for EMQX (opensource or enterprise). + +set -euo pipefail + +if [ "${DEBUG:-}" = 1 ]; then + set -x +fi + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." + +usage() { + cat </dev/null | xargs -0)" + for c in $CHECKS; do + logmsg "Executing $c" + $c + done +fi + +if [ "$DRYRUN" = 'yes' ]; then + logmsg "Release tag is ready to be created with command: git tag $TAG" +else + git tag "$TAG" + logmsg "$TAG is created OK." +fi diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh new file mode 100755 index 000000000..dda910785 --- /dev/null +++ b/scripts/rel/sync-remotes.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." + +BASE_BRANCHES=( 'release-50' 'master' ) + +usage() { + cat < Date: Mon, 28 Nov 2022 08:38:57 +0100 Subject: [PATCH 19/45] ci: add separate manual workflow to upload helm charts --- .github/workflows/release.yaml | 9 ---- .github/workflows/upload-helm-charts.yaml | 52 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/upload-helm-charts.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3a4f32499..3c157cf31 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -90,12 +90,3 @@ jobs: -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi - - uses: emqx/push-helm-action@v1 - if: github.event_name == 'release' || inputs.publish_release_artefacts - with: - charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}" - version: ${{ steps.profile.outputs.version }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-west-2" - aws_bucket_name: "repos-emqx-io" diff --git a/.github/workflows/upload-helm-charts.yaml b/.github/workflows/upload-helm-charts.yaml new file mode 100644 index 000000000..319b50e24 --- /dev/null +++ b/.github/workflows/upload-helm-charts.yaml @@ -0,0 +1,52 @@ +name: Upload helm charts +on: + release: + types: + - published + workflow_dispatch: + inputs: + tag: + type: string + required: true + +jobs: + upload: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + steps: + - uses: aws-actions/configure-aws-credentials@v1-node16 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.tag }} + - name: Detect profile + id: profile + run: | + if git describe --tags --match '[v|e]*' --exact; then + REF=$(git describe --tags --match '[v|e]*' --exact) + else + echo "Only release tags matching '[v|e]*' are supported" + exit 1 + fi + case "$REF" in + v*) + echo "profile=emqx" >> $GITHUB_OUTPUT + echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT + ;; + e*) + echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT + echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT + ;; + esac + - uses: emqx/push-helm-action@v1 + with: + charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}" + version: ${{ steps.profile.outputs.version }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-west-2" + aws_bucket_name: "repos-emqx-io" From 95faf56077555635edbc8108fe77d1abb01d86d2 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Wed, 23 Nov 2022 10:50:25 +0100 Subject: [PATCH 20/45] feat: remove $queue in favor of $shared This commit removes support for setting shared subscriptions with the non-standard $queue feature. Shared subscriptions is now part of the MQTT spec (using $shared) and we will only support that from now on. --- apps/emqx/include/emqx.hrl | 5 +---- apps/emqx/src/emqx_topic.erl | 4 ---- apps/emqx/test/emqx_proper_types.erl | 2 +- apps/emqx/test/emqx_topic_SUITE.erl | 10 ---------- .../src/emqx_mgmt_api_subscriptions.erl | 2 -- 5 files changed, 2 insertions(+), 21 deletions(-) diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index b776adab4..b1b1e2a00 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -40,15 +40,12 @@ -define(ERTS_MINIMUM_REQUIRED, "10.0"). %%-------------------------------------------------------------------- -%% Topics' prefix: $SYS | $queue | $share +%% Topics' prefix: $SYS | $share %%-------------------------------------------------------------------- %% System topic -define(SYSTOP, <<"$SYS/">>). -%% Queue topic --define(QUEUE, <<"$queue/">>). - %%-------------------------------------------------------------------- %% alarms %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 0f0a8bf1c..2941a51fe 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -210,12 +210,8 @@ parse({TopicFilter, Options}) when is_binary(TopicFilter) -> parse(TopicFilter, Options). -spec parse(topic(), map()) -> {topic(), map()}. -parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) -> - error({invalid_topic_filter, TopicFilter}); parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) -> error({invalid_topic_filter, TopicFilter}); -parse(<<"$queue/", TopicFilter/binary>>, Options) -> - parse(TopicFilter, Options#{share => <<"$queue">>}); parse(TopicFilter = <<"$share/", Rest/binary>>, Options) -> case binary:split(Rest, <<"/">>) of [_Any] -> diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 78fab0b38..0be10f476 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -424,7 +424,7 @@ systopic_mon() -> sharetopic() -> ?LET( {Type, Grp, T}, - {oneof([<<"$queue">>, <<"$share">>]), list(latin_char()), normal_topic()}, + {oneof([<<"$share">>]), list(latin_char()), normal_topic()}, <> ). diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index 13383b69e..ff0ec92a0 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -186,10 +186,6 @@ t_systop(_) -> ?assertEqual(SysTop2, systop(<<"abc">>)). t_feed_var(_) -> - ?assertEqual( - <<"$queue/client/clientId">>, - feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>) - ), ?assertEqual( <<"username/test/client/x">>, feed_var( @@ -211,10 +207,6 @@ long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]). t_parse(_) -> - ?assertError( - {invalid_topic_filter, <<"$queue/t">>}, - parse(<<"$queue/t">>, #{share => <<"g">>}) - ), ?assertError( {invalid_topic_filter, <<"$share/g/t">>}, parse(<<"$share/g/t">>, #{share => <<"g">>}) @@ -229,11 +221,9 @@ t_parse(_) -> ), ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)), ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})), - ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)), ?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)), %% The '$local' and '$fastlane' topics have been deprecated. ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)), - ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)), ?assertEqual({<<"$local/$share/group/a/b/c">>, #{}}, parse(<<"$local/$share/group/a/b/c">>)), ?assertEqual({<<"$fastlane/topic">>, #{}}, parse(<<"$fastlane/topic">>)). diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 03b833e84..9053b5339 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -182,8 +182,6 @@ format({_Subscriber, Topic, Options}) -> maps:with([qos, nl, rap, rh], Options) ). -get_topic(Topic, #{share := <<"$queue">> = Group}) -> - filename:join([Group, Topic]); get_topic(Topic, #{share := Group}) -> filename:join([<<"$share">>, Group, Topic]); get_topic(Topic, _) -> From 2582c1713e6b751cbd02c4234223a4be18a48229 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 24 Nov 2022 11:26:01 +0100 Subject: [PATCH 21/45] fix: emqx_topic:join/1 instead of filename:join/1 filename:join/1 doesn't work on Windows for joining a topic. We now use emqx_topic:join/1 instead. --- apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 9053b5339..062b915b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -183,7 +183,7 @@ format({_Subscriber, Topic, Options}) -> ). get_topic(Topic, #{share := Group}) -> - filename:join([<<"$share">>, Group, Topic]); + emqx_topic:join([<<"$share">>, Group, Topic]); get_topic(Topic, _) -> Topic. From 65398cfad7e28a4484e89c8db857317ef790e786 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 24 Nov 2022 11:48:33 +0100 Subject: [PATCH 22/45] chore: update changes --- changes/v5.0.12-en.md | 3 +++ changes/v5.0.12-zh.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index bb3df6bc7..f1cace198 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -8,4 +8,7 @@ - Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). +- Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). + Shared subscriptions are now part of the MQTT spec. Use `$shared` instead. + ## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index fad19995e..38ba95dfc 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -4,6 +4,9 @@ - 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 +- Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). + Shared subscriptions are now part of the MQTT spec. Use `$shared` instead. + ## 修复 - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 From 5e303f7abaeeb96fd7c7535188b5100bf9e09d6c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 15:04:14 +0100 Subject: [PATCH 23/45] ci: delete auto sync from emqx.git to emqx-enterprise.git --- .github/workflows/git_sync.yaml | 45 --------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 .github/workflows/git_sync.yaml diff --git a/.github/workflows/git_sync.yaml b/.github/workflows/git_sync.yaml deleted file mode 100644 index 4d81bf055..000000000 --- a/.github/workflows/git_sync.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Sync to enterprise - -on: - schedule: - - cron: '0 */6 * * *' - push: - branches: - - main-v* - -jobs: - sync_to_enterprise: - runs-on: ubuntu-20.04 - if: github.repository == 'emqx/emqx' - steps: - - name: git-sync - uses: Rory-Z/git-sync@v3.0.1 - with: - source_repo: ${{ github.repository }} - source_branch: ${{ github.ref }} - destination_repo: "${{ github.repository_owner }}/emqx-enterprise" - destination_branch: ${{ github.ref }} - destination_ssh_private_key: "${{ secrets.CI_SSH_PRIVATE_KEY }}" - - name: create pull request - id: create_pull_request - run: | - set -euo pipefail - EE_REF="${GITHUB_REF}-enterprise" - R=$(curl --silent --show-error \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -X POST \ - -d "{\"title\": \"Sync code from opensource $GITHUB_REF to entperprise $EE_REF\", \"head\": \"$GITHUB_REF\", \"base\":\"$EE_REF\"}" \ - https://api.github.com/repos/${{ github.repository_owner }}/emqx-enterprise/pulls) - echo $R | jq - echo "::set-output name=url::$(echo $R | jq '.url')" - - name: request reviewers for a pull request - if: steps.create_pull_request.outputs.url != 'null' - run: | - set -euo pipefail - curl --silent --show-error \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -X POST \ - -d '{"team_reviewers":["emqx-devs"]}' \ - ${{ steps.create_pull_request.outputs.url }}/requested_reviewers From 6fc6a23de2c97b61f6c0652d618b755bb57865fe Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Mon, 28 Nov 2022 15:17:53 +0100 Subject: [PATCH 24/45] chore: update changes Co-authored-by: Zaiming (Stone) Shi --- changes/v5.0.12-en.md | 2 +- changes/v5.0.12-zh.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index f1cace198..5c6f86832 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -9,6 +9,6 @@ - Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). - Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). - Shared subscriptions are now part of the MQTT spec. Use `$shared` instead. + Shared subscriptions are now part of the MQTT spec. Use `$share` instead. ## Bug fixes diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 38ba95dfc..53d8819e9 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -4,8 +4,8 @@ - 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 -- Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). - Shared subscriptions are now part of the MQTT spec. Use `$shared` instead. +- 删除了老的共享订阅支持方式， 不再使用 `$queue` 前缀 [#9412](https://github.com/emqx/emqx/pull/9412)。 + 共享订阅自 MQTT v5.0 开始已成为协议标准，可以使用 `$share` 前缀代替 `$queue`。 ## 修复 From 77341e7a3cf2083b3e37cfe068c41f9a9fc12d6b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 21:06:33 +0100 Subject: [PATCH 25/45] chore: bump app versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 3d1fe32d3..e687e9905 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 24cb76630..6376af6ff 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index ab726cbb2..a74d411f9 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, From f9bc700498ad73e5105c7cac086aee7aade12aa0 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 28 Nov 2022 16:47:57 +0800 Subject: [PATCH 26/45] fix(exhook): fix obsolete SSL files aren't deleted after the Exhook config update --- apps/emqx_exhook/.gitignore | 1 + apps/emqx_exhook/src/emqx_exhook_mgr.erl | 37 +++++++++++- apps/emqx_exhook/test/data/certs/cafile | 29 +++++++++ apps/emqx_exhook/test/data/certs/certfile | 24 ++++++++ apps/emqx_exhook/test/data/certs/keyfile | 27 +++++++++ apps/emqx_exhook/test/data/certs/keyfile2 | 27 +++++++++ apps/emqx_exhook/test/emqx_exhook_SUITE.erl | 66 +++++++++++++++++++++ 7 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_exhook/test/data/certs/cafile create mode 100644 apps/emqx_exhook/test/data/certs/certfile create mode 100644 apps/emqx_exhook/test/data/certs/keyfile create mode 100644 apps/emqx_exhook/test/data/certs/keyfile2 diff --git a/apps/emqx_exhook/.gitignore b/apps/emqx_exhook/.gitignore index c2e3ce6ab..83a6c6cbd 100644 --- a/apps/emqx_exhook/.gitignore +++ b/apps/emqx_exhook/.gitignore @@ -19,6 +19,7 @@ rebar3.crashdump *~ rebar.lock data/ +!test/data/ *.conf.rendered *.pyc .DS_Store diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index e58555ca1..57f3413a9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -183,8 +183,10 @@ pre_config_update(_, {enable, Name, Enable}, OldConf) -> NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)} end. -post_config_update(_KeyPath, UpdateReq, NewConf, _OldConf, _AppEnvs) -> - {ok, call({update_config, UpdateReq, NewConf})}. +post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) -> + Result = call({update_config, UpdateReq, NewConf}), + try_clear_ssl_files(UpdateReq, NewConf, OldConf), + {ok, Result}. %%===================================================================== @@ -600,3 +602,34 @@ new_ssl_source(Source, undefined) -> Source; new_ssl_source(Source, SSL) -> Source#{<<"ssl">> => SSL}. + +try_clear_ssl_files({delete, Name}, _NewConf, OldConfs) -> + OldSSL = find_server_ssl_cfg(Name, OldConfs), + emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), undefined, OldSSL); +try_clear_ssl_files({Op, Name, _}, NewConfs, OldConfs) when + Op =:= update; Op =:= enable +-> + NewSSL = find_server_ssl_cfg(Name, NewConfs), + OldSSL = find_server_ssl_cfg(Name, OldConfs), + emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), NewSSL, OldSSL); +try_clear_ssl_files(_Req, _NewConf, _OldConf) -> + ok. + +search_server_cfg(Name, Confs) -> + lists:search( + fun + (#{name := SvrName}) when SvrName =:= Name -> + true; + (_) -> + false + end, + Confs + ). + +find_server_ssl_cfg(Name, Confs) -> + case search_server_cfg(Name, Confs) of + {value, Value} -> + maps:get(ssl, Value, undefined); + false -> + undefined + end. diff --git a/apps/emqx_exhook/test/data/certs/cafile b/apps/emqx_exhook/test/data/certs/cafile new file mode 100644 index 000000000..8a9dafccd --- /dev/null +++ b/apps/emqx_exhook/test/data/certs/cafile @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF +TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy +MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe +MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0 +WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf +7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE +EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl +A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg +O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW +zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj +LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf +50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF +ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS +x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC +AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0 +e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF +KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s +tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT +WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr +c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh +kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM +kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9 +0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0 +5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn +wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb +T+fp6tNL6LE= +-----END CERTIFICATE----- diff --git a/apps/emqx_exhook/test/data/certs/certfile b/apps/emqx_exhook/test/data/certs/certfile new file mode 100644 index 000000000..a198faf61 --- /dev/null +++ b/apps/emqx_exhook/test/data/certs/certfile @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/jCCAeagAwIBAgIJAKTICmq1Lg6dMA0GCSqGSIb3DQEBCwUAMDQxEjAQBgNV +BAoMCUVNUVggVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4X +DTIxMTIzMDA4NDExMloXDTQ5MDUxNzA4NDExMlowJTESMBAGA1UECgwJRU1RWCBU +ZXN0MQ8wDQYDVQQDDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDzrujfx6XZTH0MWqLO6kNAeHndUZ+OGaURXvxKMPMF5dA40lxNG6cEzzlq +0Rm61adlv8tF4kRJrs6EnRjEVoMImrdh07vGFdOTYqP01LjiBhErAzyRtSn2X8FT +Te8ExoCRs3x61SPebGY2hOvFxuO6YDPVOSDvbbxvRgqIlM1ZXC8dOvPSSGZ+P8hV +56EPayRthfu1FVptnkW9CyZCRI0gg95Hv8RC7bGG+tuWpkN9ZrRvohhgGR1+bDUi +BNBpncEsSh+UgWaj8KRN8D16H6m/Im6ty467j0at49FvPx5nACL48/ghtYvzgKLc +uKHtokKUuuzebDK/hQxN3mUSAJStAgMBAAGjIjAgMAsGA1UdDwQEAwIFoDARBglg +hkgBhvhCAQEEBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAIlVyPhOpkz3MNzQmjX7 +xgJ3vGPK5uK11n/wfjRwe2qXwZbrI2sYLVtTpUgvLDuP0gB73Vwfu7xAMdue6TRm +CKr9z0lkQsVBtgoqzZCjd4PYLfHm4EhsOMi98OGKU5uOGD4g3yLwQWXHhbYtiZMO +Jsj0hebYveYJt/BYTd1syGQcIcYCyVExWvSWjidfpAqjT6EF7whdubaFtuF2kaGF +IO9yn9rWtXB5yK99uCguEmKhx3fAQxomzqweTu3WRvy9axsUH3WAUW9a4DIBSz2+ +ZSJNheFn5GktgggygJUGYqpSZHooUJW0UBs/8vX6AP+8MtINmqOGZUawmNwLWLOq +wHyVt2YGD5TXjzzsWNSQ4mqXxM6AXniZVZK0yYNjA4ATikX1AtwunyWBR4IjyE/D +FxYPORdZCOtywRFE1R5KLTUq/C8BNGCkYnoO78DJBO+pT0oagkQGQb0CnmC6C1db +4lWzA9K0i4B0PyooZA+gp+5FFgaLuX1DkyeaY1J204QhHR1z/Vcyl5dpqR9hqnYP +t8raLk9ogMDKqKA9iG0wc3CBNckD4sjVWAEeovXhElG55fD21wwhF+AnDCvX8iVK +cBfKV6z6uxfKjGIxc2I643I5DiIn+V3DnPxYyY74Ln1lWFYmt5JREhAxPu42zq74 +e6+eIMYFszB+5gKgt6pa6ZNI +-----END CERTIFICATE----- diff --git a/apps/emqx_exhook/test/data/certs/keyfile b/apps/emqx_exhook/test/data/certs/keyfile new file mode 100644 index 000000000..2f0af5d41 --- /dev/null +++ b/apps/emqx_exhook/test/data/certs/keyfile @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA867o38el2Ux9DFqizupDQHh53VGfjhmlEV78SjDzBeXQONJc +TRunBM85atEZutWnZb/LReJESa7OhJ0YxFaDCJq3YdO7xhXTk2Kj9NS44gYRKwM8 +kbUp9l/BU03vBMaAkbN8etUj3mxmNoTrxcbjumAz1Tkg7228b0YKiJTNWVwvHTrz +0khmfj/IVeehD2skbYX7tRVabZ5FvQsmQkSNIIPeR7/EQu2xhvrblqZDfWa0b6IY +YBkdfmw1IgTQaZ3BLEoflIFmo/CkTfA9eh+pvyJurcuOu49GrePRbz8eZwAi+PP4 +IbWL84Ci3Lih7aJClLrs3mwyv4UMTd5lEgCUrQIDAQABAoIBAQDwEbBgznrIwn8r +jZt5x/brbAV7Ea/kOcWSgIaCvQifFdJ2OGAwov5/UXwajNgRZe2d4z7qoUhvYuUY +ZwCAZU6ASpRBr2v9cYFYYURvrqZaHmoJew3P6q/lhl6aqFvC06DUagRHqvXEafyk +13zEAvZVpfNKrBaTawPKiDFWb2qDDc9D6hC07EuJ/DNeehiHvzHrSZSDVV5Ut7Bw +YDm33XygheUPAlHfeCnaixzcs3osiVyFEmVjxcIaM0ZS1NgcSaohSpJHMzvEaohX +e+v9vccraSVlw01AlvFwI2vHYUV8jT6HwglTPKKGOCzK/ace3wPdYSU9qLcqfuHn +EFhNc3tNAoGBAPugLMgbReJg2gpbIPUkYyoMMAAU7llFU1WvPWwXzo1a9EBjBACw +WfCZISNtANXR38zIYXzoH547uXi4YPks1Nne3sYuCDpvuX+iz7fIo4zHf1nFmxH7 +eE6GtQr2ubmuuipTc28S0wBMGT1/KybH0e2NKL6GaOkNDmAI0IbEMBrvAoGBAPfr +Y1QYLhPhan6m5g/5s+bQpKtHfNH9TNkk13HuYu72zNuY3qL2GC7oSadR8vTbRXZg +KQqfaO0IGRcdkSFTq/AEhSSqr2Ld5nPadMbKvSGrSCc1s8rFH97jRVQY56yhM7ti +IW4+6cE8ylCMbdYB6wuduK/GIgNpqoF4xs1i2XojAoGACacBUMPLEH4Kny8TupOk +wi4pgTdMVVxVcAoC3yyincWJbRbfRm99Y79cCBHcYFdmsGJXawU0gUtlN/5KqgRQ +PfNQtGV7p1I12XGTakdmDrZwai8sXao52TlNpJgGU9siBRGicfZU5cQFi9he/WPY +57XshDJ/v8DidkigRysrdT0CgYEA5iuO22tblC+KvK1dGOXeZWO+DhrfwuGlcFBp +CaimB2/w/8vsn2VVTG9yujo2E6hj1CQw1mDrfG0xRim4LTXOgpbfugwRqvuTUmo2 +Ur21XEX2RhjwpEfhcACWxB4fMUG0krrniMA2K6axupi1/KNpQi6bYe3UdFCs8Wld +QSAOAvsCgYBk/X5PmD44DvndE5FShM2w70YOoMr3Cgl5sdwAFUFE9yDuC14UhVxk +oxnYxwtVI9uVVirET+LczP9JEvcvxnN/Xg3tH/qm0WlIxmTxyYrFFIK9j0rqeu9z +blPu56OzNI2VMrR1GbOBLxQINLTIpaacjNJAlr8XOlegdUJsW/Jwqw== +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_exhook/test/data/certs/keyfile2 b/apps/emqx_exhook/test/data/certs/keyfile2 new file mode 100644 index 000000000..2b3f30cf6 --- /dev/null +++ b/apps/emqx_exhook/test/data/certs/keyfile2 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzLiGiSwpxkENtjrzS7pNLblTnWe4HUUFwYyUX0H+3TnvA86X +EX85yZvFjkzB6lLjUkMY+C6UTVXt+mxeSJbUtSKZhX+2yoF/KYh7SaVjug5FqEqO +LvMpDZQEhUWF2W9DG6eUgOfDoX2milSDIe10yG2WBkryipHAfE7l1t+i6Rh3on+v +561LmrbqyBWR/cLp23RN3sHbkf2pb5/ugtU9twdgJr6Lve73rvSeulewL5BzszKD +BrYqr+PBT5+3ItCc55bTsO7M7CzOIL99BlqdvFH7xT0U1+2BFwLe4/8kwphSqyJE +C5oOiQBFnFVNXmFQSV+k7rPr80i1IO++HeJ6KQIDAQABAoIBAGWgvPjfuaU3qizq +uti/FY07USz0zkuJdkANH6LiSjlchzDmn8wJ0pApCjuIE0PV/g9aS8z4opp5q/gD +UBLM/a8mC/xf2EhTXOMrY7i9p/I3H5FZ4ZehEqIw9sWKK9YzC6dw26HabB2BGOnW +5nozPSQ6cp2RGzJ7BIkxSZwPzPnVTgy3OAuPOiJytvK+hGLhsNaT+Y9bNDvplVT2 +ZwYTV8GlHZC+4b2wNROILm0O86v96O+Qd8nn3fXjGHbMsAnONBq10bZS16L4fvkH +5G+W/1PeSXmtZFppdRRDxIW+DWcXK0D48WRliuxcV4eOOxI+a9N2ZJZZiNLQZGwg +w3A8+mECgYEA8HuJFrlRvdoBe2U/EwUtG74dcyy30L4yEBnN5QscXmEEikhaQCfX +Wm6EieMcIB/5I5TQmSw0cmBMeZjSXYoFdoI16/X6yMMuATdxpvhOZGdUGXxhAH+x +xoTUavWZnEqW3fkUU71kT5E2f2i+0zoatFESXHeslJyz85aAYpP92H0CgYEA2e5A +Yozt5eaA1Gyhd8SeptkEU4xPirNUnVQHStpMWUb1kzTNXrPmNWccQ7JpfpG6DcYl +zUF6p6mlzY+zkMiyPQjwEJlhiHM2NlL1QS7td0R8ewgsFoyn8WsBI4RejWrEG9td +EDniuIw+pBFkcWthnTLHwECHdzgquToyTMjrBB0CgYEA28tdGbrZXhcyAZEhHAZA +Gzog+pKlkpEzeonLKIuGKzCrEKRecIK5jrqyQsCjhS0T7ZRnL4g6i0s+umiV5M5w +fcc292pEA1h45L3DD6OlKplSQVTv55/OYS4oY3YEJtf5mfm8vWi9lQeY8sxOlQpn +O+VZTdBHmTC8PGeTAgZXHZUCgYA6Tyv88lYowB7SN2qQgBQu8jvdGtqhcs/99GCr +H3N0I69LPsKAR0QeH8OJPXBKhDUywESXAaEOwS5yrLNP1tMRz5Vj65YUCzeDG3kx +gpvY4IMp7ArX0bSRvJ6mYSFnVxy3k174G3TVCfksrtagHioVBGQ7xUg5ltafjrms +n8l55QKBgQDVzU8tQvBVqY8/1lnw11Vj4fkE/drZHJ5UkdC1eenOfSWhlSLfUJ8j +ds7vEWpRPPoVuPZYeR1y78cyxKe1GBx6Wa2lF5c7xjmiu0xbRnrxYeLolce9/ntp +asClqpnHT8/VJYTD7Kqj0fouTTZf0zkig/y+2XERppd8k+pSKjUCPQ== +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index a0472b2c3..259e1ced8 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -348,6 +348,55 @@ t_stop_timeout(_) -> snabbkaffe:stop(), meck:unload(emqx_exhook_demo_svr). +t_ssl_clear(_) -> + SvrName = <<"ssl_test">>, + SSLConf = #{ + <<"cacertfile">> => cert_file("cafile"), + + <<"certfile">> => cert_file("certfile"), + + <<"enable">> => true, + <<"keyfile">> => cert_file("keyfile"), + + <<"verify">> => <<"verify_peer">> + }, + AddConf = #{ + <<"auto_reconnect">> => <<"60s">>, + <<"enable">> => false, + <<"failed_action">> => <<"deny">>, + <<"name">> => <<"ssl_test">>, + <<"pool_size">> => 16, + <<"request_timeout">> => <<"5s">>, + <<"ssl">> => SSLConf, + + <<"url">> => <<"http://127.0.0.1:9000">> + }, + emqx_exhook_mgr:update_config([exhook, servers], {add, AddConf}), + ListResult1 = list_pem_dir(SvrName), + ?assertMatch({ok, [_, _, _]}, ListResult1), + {ok, ResultList1} = ListResult1, + + UpdateConf = AddConf#{<<"ssl">> => SSLConf#{<<"keyfile">> => cert_file("keyfile2")}}, + emqx_exhook_mgr:update_config([exhook, servers], {update, SvrName, UpdateConf}), + ListResult2 = list_pem_dir(SvrName), + ?assertMatch({ok, [_, _, _]}, ListResult2), + {ok, ResultList2} = ListResult2, + + FindKeyFile = fun(List) -> + case lists:search(fun(E) -> lists:prefix("key", E) end, List) of + {value, Value} -> + Value; + _ -> + ?assert(false, "Can't find keyfile") + end + end, + + ?assertNotEqual(FindKeyFile(ResultList1), FindKeyFile(ResultList2)), + + emqx_exhook_mgr:update_config([exhook, servers], {delete, SvrName}), + ?assertMatch({error, not_dir}, list_pem_dir(SvrName)), + ok. + %%-------------------------------------------------------------------- %% Cases Helpers %%-------------------------------------------------------------------- @@ -414,3 +463,20 @@ loaded_exhook_hookpoints() -> is_exhook_callback(Cb) -> Action = element(2, Cb), emqx_exhook_handler == element(1, Action). + +list_pem_dir(Name) -> + Dir = filename:join([emqx:mutable_certs_dir(), "exhook", Name]), + case filelib:is_dir(Dir) of + true -> + file:list_dir(Dir); + _ -> + {error, not_dir} + end. + +data_file(Name) -> + Dir = code:lib_dir(emqx_exhook, test), + {ok, Bin} = file:read_file(filename:join([Dir, "data", Name])), + Bin. + +cert_file(Name) -> + data_file(filename:join(["certs", Name])). From 606389edd2e5401e6cbb3e2652ff5efb1bf1171d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 29 Nov 2022 10:41:21 +0800 Subject: [PATCH 27/45] chore: update changes && bump version --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 4e8abef81..4e8d3d514 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 5c6f86832..da20798a5 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -12,3 +12,5 @@ Shared subscriptions are now part of the MQTT spec. Use `$share` instead. ## Bug fixes + +- Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 53d8819e9..45acfd751 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -7,8 +7,10 @@ - 删除了老的共享订阅支持方式， 不再使用 `$queue` 前缀 [#9412](https://github.com/emqx/emqx/pull/9412)。 共享订阅自 MQTT v5.0 开始已成为协议标准，可以使用 `$share` 前缀代替 `$queue`。 -## 修复 - - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 - 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 + +## 修复 + +- 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 From b124e64399bd9f138cf9f85100e2c2291ecf03cf Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 24 Nov 2022 16:57:36 +0100 Subject: [PATCH 28/45] refactor: use PUT on `/position` to move authenticator --- apps/emqx_authn/i18n/emqx_authn_api_i18n.conf | 12 ++- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authn/src/emqx_authn_api.erl | 99 +++++++------------ apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 64 ++++++------ changes/v5.0.12-en.md | 3 + changes/v5.0.12-zh.md | 8 +- 6 files changed, 80 insertions(+), 108 deletions(-) diff --git a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf index f42ff7417..da5d6a1d1 100644 --- a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf +++ b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf @@ -84,14 +84,14 @@ emqx_authn_api { } } - authentication_id_move_post { + authentication_id_position_put { desc { en: """Move authenticator in global authentication chain.""" zh: """更改全局认证链上指定认证器的顺序。""" } } - listeners_listener_id_authentication_id_move_post { + listeners_listener_id_authentication_id_position_put { desc { en: """Move authenticator in listener authentication chain.""" zh: """更改监听器认证链上指定认证器的顺序。""" @@ -182,7 +182,6 @@ emqx_authn_api { } } - param_user_id { desc { en: """User ID.""" @@ -190,6 +189,13 @@ emqx_authn_api { } } + param_position { + desc { + en: """Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" + zn: """认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'""" + } + } + like_user_id { desc { en: """Fuzzy search user_id (username or clientid).""" diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 9f66a978c..5c27c4b3b 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index f6627b3be..ee8757774 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -55,8 +55,8 @@ listener_authenticators/2, listener_authenticator/2, listener_authenticator_status/2, - authenticator_move/2, - listener_authenticator_move/2, + authenticator_position/2, + listener_authenticator_position/2, authenticator_users/2, authenticator_user/2, listener_authenticator_users/2, @@ -67,7 +67,6 @@ -export([ authenticator_examples/0, - request_move_examples/0, request_user_create_examples/0, request_user_update_examples/0, response_user_examples/0, @@ -99,14 +98,14 @@ paths() -> "/authentication", "/authentication/:id", "/authentication/:id/status", - "/authentication/:id/move", + "/authentication/:id/position/:position", "/authentication/:id/users", "/authentication/:id/users/:user_id", "/listeners/:listener_id/authentication", "/listeners/:listener_id/authentication/:id", "/listeners/:listener_id/authentication/:id/status", - "/listeners/:listener_id/authentication/:id/move", + "/listeners/:listener_id/authentication/:id/position/:position", "/listeners/:listener_id/authentication/:id/users", "/listeners/:listener_id/authentication/:id/users/:user_id" ]. @@ -115,7 +114,6 @@ roots() -> [ request_user_create, request_user_update, - request_move, response_user, response_users ]. @@ -130,8 +128,6 @@ fields(request_user_update) -> {password, mk(binary(), #{required => true})}, {is_superuser, mk(boolean(), #{default => false, required => false})} ]; -fields(request_move) -> - [{position, mk(binary(), #{required => true})}]; fields(response_user) -> [ {user_id, mk(binary(), #{required => true})}, @@ -321,17 +317,13 @@ schema("/listeners/:listener_id/authentication/:id/status") -> } } }; -schema("/authentication/:id/move") -> +schema("/authentication/:id/position/:position") -> #{ - 'operationId' => authenticator_move, - post => #{ + 'operationId' => authenticator_position, + put => #{ tags => ?API_TAGS_GLOBAL, - description => ?DESC(authentication_id_move_post), - parameters => [param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_move), - request_move_examples() - ), + description => ?DESC(authentication_id_position_put), + parameters => [param_auth_id(), param_position()], responses => #{ 204 => <<"Authenticator moved">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -339,17 +331,13 @@ schema("/authentication/:id/move") -> } } }; -schema("/listeners/:listener_id/authentication/:id/move") -> +schema("/listeners/:listener_id/authentication/:id/position/:position") -> #{ - 'operationId' => listener_authenticator_move, - post => #{ + 'operationId' => listener_authenticator_position, + put => #{ tags => ?API_TAGS_SINGLE, - description => ?DESC(listeners_listener_id_authentication_id_move_post), - parameters => [param_listener_id(), param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_move), - request_move_examples() - ), + description => ?DESC(listeners_listener_id_authentication_id_position_put), + parameters => [param_listener_id(), param_auth_id(), param_position()], responses => #{ 204 => <<"Authenticator moved">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -556,6 +544,17 @@ param_listener_id() -> }) }. +param_position() -> + { + position, + mk(binary(), #{ + in => path, + desc => ?DESC(param_position), + required => true, + example => "before:password_based:built_in_database" + }) + }. + param_user_id() -> { user_id, @@ -662,23 +661,15 @@ listener_authenticator_status( end ). -authenticator_move( - post, - #{ - bindings := #{id := AuthenticatorID}, - body := #{<<"position">> := Position} - } +authenticator_position( + put, + #{bindings := #{id := AuthenticatorID, position := Position}} ) -> - move_authenticator([authentication], ?GLOBAL, AuthenticatorID, Position); -authenticator_move(post, #{bindings := #{id := _}, body := _}) -> - serialize_error({missing_parameter, position}). + move_authenticator([authentication], ?GLOBAL, AuthenticatorID, Position). -listener_authenticator_move( - post, - #{ - bindings := #{listener_id := ListenerID, id := AuthenticatorID}, - body := #{<<"position">> := Position} - } +listener_authenticator_position( + put, + #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, position := Position}} ) -> with_listener( ListenerID, @@ -690,9 +681,7 @@ listener_authenticator_move( Position ) end - ); -listener_authenticator_move(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> - serialize_error({missing_parameter, position}). + ). authenticator_users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> add_user(?GLOBAL, AuthenticatorID, UserInfo); @@ -1475,28 +1464,6 @@ request_user_update_examples() -> } }. -request_move_examples() -> - #{ - move_to_front => #{ - summary => <<"Move authenticator to the beginning of the chain">>, - value => #{ - position => <<"front">> - } - }, - move_to_rear => #{ - summary => <<"Move authenticator to the end of the chain">>, - value => #{ - position => <<"rear">> - } - }, - 'move_before_password_based:built_in_database' => #{ - summary => <<"Move authenticator to the position preceding some other authenticator">>, - value => #{ - position => <<"before:password_based:built_in_database">> - } - } - }. - response_user_examples() -> #{ regular_user => #{ diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 64247f2bc..a1047e25c 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -120,8 +120,8 @@ t_authenticator_users(_) -> t_authenticator_user(_) -> test_authenticator_user([]). -t_authenticator_move(_) -> - test_authenticator_move([]). +t_authenticator_position(_) -> + test_authenticator_position([]). t_authenticator_import_users(_) -> test_authenticator_import_users([]). @@ -138,8 +138,8 @@ t_listener_authenticator_users(_) -> t_listener_authenticator_user(_) -> test_authenticator_user(["listeners", ?TCP_DEFAULT]). -t_listener_authenticator_move(_) -> - test_authenticator_move(["listeners", ?TCP_DEFAULT]). +t_listener_authenticator_position(_) -> + test_authenticator_position(["listeners", ?TCP_DEFAULT]). t_listener_authenticator_import_users(_) -> test_authenticator_import_users(["listeners", ?TCP_DEFAULT]). @@ -539,7 +539,7 @@ test_authenticator_user(PathPrefix) -> {ok, 404, _} = request(delete, UsersUri ++ "/u123"), {ok, 204, _} = request(delete, UsersUri ++ "/u1"). -test_authenticator_move(PathPrefix) -> +test_authenticator_position(PathPrefix) -> AuthenticatorConfs = [ emqx_authn_test_lib:http_example(), emqx_authn_test_lib:jwt_example(), @@ -569,42 +569,31 @@ test_authenticator_move(PathPrefix) -> %% Invalid moves {ok, 400, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"up">>} - ), - - {ok, 400, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "up"]) ), {ok, 404, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"before:invalid">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position"]) ), {ok, 404, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"before:password_based:redis">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "before:invalid"]) ), {ok, 404, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"before:password_based:redis">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "before:password_based:redis"]) ), %% Valid moves %% test front {ok, 204, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"front">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "front"]) ), ?assertAuthenticatorsMatch( @@ -618,9 +607,8 @@ test_authenticator_move(PathPrefix) -> %% test rear {ok, 204, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"rear">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "rear"]) ), ?assertAuthenticatorsMatch( @@ -634,9 +622,8 @@ test_authenticator_move(PathPrefix) -> %% test before {ok, 204, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), - #{position => <<"before:password_based:built_in_database">>} + put, + uri(PathPrefix ++ [?CONF_NS, "jwt", "position", "before:password_based:built_in_database"]) ), ?assertAuthenticatorsMatch( @@ -650,9 +637,16 @@ test_authenticator_move(PathPrefix) -> %% test after {ok, 204, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS, "password_based%3Abuilt_in_database", "move"]), - #{position => <<"after:password_based:http">>} + put, + uri( + PathPrefix ++ + [ + ?CONF_NS, + "password_based%3Abuilt_in_database", + "position", + "after:password_based:http" + ] + ) ), ?assertAuthenticatorsMatch( diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index da20798a5..0828616de 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -11,6 +11,9 @@ - Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). Shared subscriptions are now part of the MQTT spec. Use `$share` instead. +- Refactor authn API by replacing `POST /authentication/{id}/move` with `PUT /authentication/{id}/position/{position}`. [#9419](https://github.com/emqx/emqx/pull/9419). + Same is done for `/listeners/{listener_id}/authentication/id/...`. + ## Bug fixes - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 45acfd751..bfe75ab36 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -4,13 +4,15 @@ - 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 -- 删除了老的共享订阅支持方式， 不再使用 `$queue` 前缀 [#9412](https://github.com/emqx/emqx/pull/9412)。 - 共享订阅自 MQTT v5.0 开始已成为协议标准，可以使用 `$share` 前缀代替 `$queue`。 - - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 - 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 +- 删除了老的共享订阅支持方式， 不再使用 `$queue` 前缀 [#9412](https://github.com/emqx/emqx/pull/9412)。 + 共享订阅自 MQTT v5.0 开始已成为协议标准，可以使用 `$share` 前缀代替 `$queue`。 + +- 重构认证 API，使用 `PUT /authentication/{id}/position/{position}` 代替了 `POST /authentication/{id}/move` [#9419](https://github.com/emqx/emqx/pull/9419)。 + ## 修复 - 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 From 6ced692ce6c07474f5e0ddfa11fccef2d246129f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 29 Nov 2022 20:18:17 +0100 Subject: [PATCH 29/45] docs: fix minor issues in docker readme --- deploy/docker/README.md | 85 +++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index d2d729b5d..51d53d1e3 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -30,11 +30,15 @@ Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol speci Execute some command under this docker image -``docker run -d --name emqx emqx/emqx:$(tag)`` +```console +$ docker run -d --name emqx emqx/emqx:${tag} +``` For example -``docker run -d --name emqx -p 18083:18083 -p 1883:1883 emqx/emqx:latest`` +```console +$ docker run -d --name emqx -p 18083:18083 -p 1883:1883 emqx/emqx:latest +``` The EMQX broker runs as Linux user `emqx` in the docker container. @@ -42,7 +46,7 @@ The EMQX broker runs as Linux user `emqx` in the docker container. Use the environment variable to configure the EMQX docker container. -By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files. +By default, the environment variables with `EMQX_` prefix are mapped to key-value pairs in configuration files. You can change the prefix by overriding `HOCON_ENV_OVERRIDE_PREFIX`. @@ -53,9 +57,9 @@ EMQX_LISTENERS__SSL__DEFAULT__ACCEPTORS <--> listeners.ssl.default.acceptors EMQX_ZONES__DEFAULT__MQTT__MAX_PACKET_SIZE <--> zones.default.mqtt.max_packet_size ``` -+ Prefix ``EMQX_`` is removed ++ Prefix `EMQX_` is removed + All upper case letters is replaced with lower case letters -+ ``__`` is replaced with ``.`` ++ `__` is replaced with `.` If `HOCON_ENV_OVERRIDE_PREFIX=DEV_` is set: @@ -75,41 +79,43 @@ These environment variables will ignore for configuration file. #### EMQX Configuration -> NOTE: All EMQX Configuration in [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) can be configured via environment variables. The following list is just an example, not a complete configuration. +> NOTE: All EMQX Configuration in [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/apps/emqx/etc/emqx.conf) can be configured via environment variables. The following list is just an example, not a complete configuration. | Options | Default | Mapped | Description | | ---------------------------| ------------------ | ------------------------- | ------------------------------------- | | `EMQX_NAME` | container name | none | EMQX node short name | | `EMQX_HOST` | container IP | none | EMQX node host, IP or FQDN | -The list is incomplete and may changed with [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. +The list is incomplete and may be changed with [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/apps/emqx/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. -If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE_NAME``, ``EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST``. +If set `EMQX_NAME` and `EMQX_HOST`, and unset `EMQX_NODE_NAME`, `EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST`. For example, set MQTT TCP port to 1883 -``docker run -d --name emqx -e EMQX__LISTENERS__TCP__DEFAULT__BIND=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest`` +```console +$ docker run -d --name emqx -e EMQX__LISTENERS__TCP__DEFAULT__BIND=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest +``` #### EMQX Loaded Modules Configuration -| Oprtions | Default | Description | -| ------------------------ | ------------------ | ------------------------------------- | -| `EMQX_LOADED_MODULES` | see content below | default EMQX loaded modules | +| Options | Default | Description | +| ----------------------- | ------------------ | ------------------------------------- | +| `EMQX_LOADED_MODULES` | see content below | default EMQX loaded modules | -Default environment variable ``EMQX_LOADED_MODULES``, including +Default environment variable `EMQX_LOADED_MODULES`, including -+ ``emqx_mod_presence`` ++ `emqx_mod_presence` ```bash # The default EMQX_LOADED_MODULES env EMQX_LOADED_MODULES="emqx_mod_presence" ``` -For example, set ``EMQX_LOADED_MODULES=emqx_mod_delayed,emqx_mod_rewrite`` to load these two modules. +For example, set `EMQX_LOADED_MODULES=emqx_mod_delayed,emqx_mod_rewrite` to load these two modules. You can use comma, space or other separator that you want. -All the modules defined in env ``EMQX_LOADED_MODULES`` will be loaded. +All the modules defined in env `EMQX_LOADED_MODULES` will be loaded. ```bash EMQX_LOADED_MODULES="emqx_mod_delayed,emqx_mod_rewrite" @@ -119,28 +125,28 @@ EMQX_LOADED_MODULES="emqx_mod_delayed | emqx_mod_rewrite" #### EMQX Loaded Plugins Configuration -| Oprtions | Default | Description | -| ------------------------ | ------------------ | ------------------------------------- | -| `EMQX_LOADED_PLUGINS` | see content below | default EMQX loaded plugins | +| Options | Default | Description | +| ----------------------- | ------------------ | ------------------------------------- | +| `EMQX_LOADED_PLUGINS` | see content below | default EMQX loaded plugins | -Default environment variable ``EMQX_LOADED_PLUGINS``, including +Default environment variable `EMQX_LOADED_PLUGINS`, including -+ ``emqx_recon`` -+ ``emqx_retainer`` -+ ``emqx_rule_engine`` -+ ``emqx_management`` -+ ``emqx_dashboard`` ++ `emqx_recon` ++ `emqx_retainer` ++ `emqx_rule_engine` ++ `emqx_management` ++ `emqx_dashboard` ```bash # The default EMQX_LOADED_PLUGINS env EMQX_LOADED_PLUGINS="emqx_recon,emqx_retainer,emqx_management,emqx_dashboard" ``` -For example, set ``EMQX_LOADED_PLUGINS= emqx_retainer,emqx_rule_engine`` to load these two plugins. +For example, set `EMQX_LOADED_PLUGINS= emqx_retainer,emqx_rule_engine` to load these two plugins. You can use comma, space or other separator that you want. -All the plugins defined in ``EMQX_LOADED_PLUGINS`` will be loaded. +All the plugins defined in `EMQX_LOADED_PLUGINS` will be loaded. ```bash EMQX_LOADED_PLUGINS="emqx_retainer,emqx_rule_engine" @@ -150,7 +156,7 @@ EMQX_LOADED_PLUGINS="emqx_retainer | emqx_rule_engine" #### EMQX Plugins Configuration -The environment variables which with ``EMQX_`` prefix are mapped to all EMQX plugins' configuration file, ``.`` get replaced by ``__``. +The environment variables which with `EMQX_` prefix are mapped to all EMQX plugins' configuration file, `.` get replaced by `__`. Example: @@ -177,7 +183,7 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 \ emqx/emqx:latest ``` -For numbered configuration options where the number is next to a ``.`` such as: +For numbered configuration options where the number is next to a `.` such as: + backend.redis.pool1.server + backend.redis.hook.message.publish.1 @@ -186,20 +192,20 @@ You can configure an arbitrary number of them as long as each has a unique numbe ```bash docker run -d --name emqx -p 18083:18083 -p 1883:1883 \ - -e EMQX_BACKEND_REDIS_POOL1__SERVER=127.0.0.1:6379 + -e EMQX_BACKEND_REDIS_POOL1__SERVER=127.0.0.1:6379 \ [...] - -e EMQX_BACKEND__REDIS__POOL5__SERVER=127.0.0.5:6379 - -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__1='{"topic": "persistent/topic1", "action": {"function": "on_message_publish"}, "pool": "pool1"}' - -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__2='{"topic": "persistent/topic2", "action": {"function": "on_message_publish"}, "pool": "pool1"}' - -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__3='{"topic": "persistent/topic3", "action": {"function": "on_message_publish"}, "pool": "pool1"}' + -e EMQX_BACKEND__REDIS__POOL5__SERVER=127.0.0.5:6379 \ + -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__1='{"topic": "persistent/topic1", "action": {"function": "on_message_publish"}, "pool": "pool1"}' \ + -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__2='{"topic": "persistent/topic2", "action": {"function": "on_message_publish"}, "pool": "pool1"}' \ + -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__3='{"topic": "persistent/topic3", "action": {"function": "on_message_publish"}, "pool": "pool1"}' \ [...] - -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__13='{"topic": "persistent/topic13", "action": {"function": "on_message_publish"}, "pool": "pool1"}' + -e EMQX_BACKEND__REDIS__HOOK_MESSAGE__PUBLISH__13='{"topic": "persistent/topic13", "action": {"function": "on_message_publish"}, "pool": "pool1"}' \ emqx/emqx:latest ``` ### Cluster -EMQX supports a variety of clustering methods, see our [documentation](https://www.emqx.io/docs/en/latest/advanced/cluster.html) for details. +EMQX supports a variety of clustering methods, see our [documentation](https://www.emqx.io/docs/en/latest/deploy/cluster/intro.html) for details. Let's create a static node list cluster from docker-compose. @@ -236,7 +242,6 @@ Let's create a static node list cluster from docker-compose. networks: emqx-bridge: driver: bridge - ``` + Start the docker-compose cluster @@ -276,7 +281,7 @@ volumes: services: emqx: - image: emqx/emqx:v4.0.0 + image: emqx/emqx:latest restart: always environment: EMQX_NAME: foo_emqx @@ -289,12 +294,11 @@ services: ### Kernel Tuning -Under Linux host machine, the easiest way is [Tuning guide](https://www.emqx.io/docs/en/latest/tutorial/tune.html). +Under Linux host machine, the easiest way is [Tuning guide](https://www.emqx.io/docs/en/latest/deploy/tune.html). If you want tune Linux kernel by docker, you must ensure your docker is latest version (>=1.12). ```bash - docker run -d --name emqx -p 18083:18083 -p 1883:1883 \ --sysctl fs.file-max=2097152 \ --sysctl fs.nr_open=2097152 \ @@ -312,7 +316,6 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 \ --sysctl net.ipv4.tcp_max_tw_buckets=1048576 \ --sysctl net.ipv4.tcp_fin_timeout=15 \ emqx/emqx:latest - ``` > REMEMBER: DO NOT RUN EMQX DOCKER PRIVILEGED OR MOUNT SYSTEM PROC IN CONTAINER TO TUNE LINUX KERNEL, IT IS UNSAFE. From 5eacc84d1f648fb9df55516a428f98e6dfc27309 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 30 Nov 2022 09:30:40 +0100 Subject: [PATCH 30/45] ci: delete otp_release from log artifact name --- .github/workflows/run_test_cases.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 03030908c..f35a7156e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -97,8 +97,6 @@ jobs: fail-fast: false matrix: app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} - otp_release: - - "erlang24" runs-on: aws-amd64 defaults: @@ -130,7 +128,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs_${{ matrix.otp_release }}-${{ matrix.profile }} + name: logs-${{ matrix.profile }} path: source/_build/test/logs ct: @@ -192,7 +190,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs_${{ matrix.otp_release }}-${{ matrix.profile }} + name: logs-${{ matrix.profile }} path: source/_build/test/logs make_cover: From 720c233dd5ead1d7aa67909497a68512f846ce64 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 30 Nov 2022 09:37:08 +0100 Subject: [PATCH 31/45] ci: delete codeball --- .github/workflows/codeball.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .github/workflows/codeball.yml diff --git a/.github/workflows/codeball.yml b/.github/workflows/codeball.yml deleted file mode 100644 index ed69d5c10..000000000 --- a/.github/workflows/codeball.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Codeball -on: [pull_request] - -jobs: - codeball_job: - runs-on: ubuntu-latest - name: Codeball - steps: - # Run Codeball on all new Pull Requests 🚀 - # For customizations and more documentation, see https://github.com/sturdy-dev/codeball-action - - name: Codeball - uses: sturdy-dev/codeball-action@v2 - with: - approvePullRequests: "true" - labelPullRequestsWhenApproved: "true" - labelPullRequestsWhenReviewNeeded: "false" - failJobsWhenReviewNeeded: "false" From 51ace31f303074fb75942303ae432a2f4ffb803a Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 30 Nov 2022 21:36:43 +0800 Subject: [PATCH 32/45] fix(connector): fix previous incorrectly fixes There was an error in the try_clear_certs2/3 clause that the argument isn't the ssl field but including it. --- apps/emqx_connector/src/emqx_connector_ssl.erl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 7dc6179e1..450206ced 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -70,14 +70,11 @@ try_clear_certs(RltvDir, NewConf, OldConf) -> ). try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) -> - NewSSL = maps:get(<<"ssl">>, NewConnector, undefined), - OldSSL = maps:get(<<"ssl">>, OldConnector, undefined), - try_clear_certs2(RltvDir, NewSSL, OldSSL); -try_clear_certs2(RltvDir, NewSSL, OldSSL) when is_map(NewSSL) andalso is_map(OldSSL) -> - ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL); + try_clear_certs2(RltvDir, NewConnector, OldConnector); try_clear_certs2(RltvDir, NewConf, OldConf) -> - ?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}), - ok. + NewSSL = try_map_get(<<"ssl">>, NewConf, undefined), + OldSSL = try_map_get(<<"ssl">>, OldConf, undefined), + ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL). new_ssl_config(RltvDir, Config, SSL) -> case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of @@ -100,3 +97,8 @@ new_ssl_config(Config, _NewSSL) -> normalize_key_to_bin(Map) -> emqx_map_lib:binary_key_map(Map). + +try_map_get(Key, Map, Default) when is_map(Map) -> + maps:get(Key, Map, Default); +try_map_get(_Key, undefined, Default) -> + Default. From 81586a4c8d6a2ddfef937bdb3fbeae752ed878ea Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 30 Nov 2022 21:48:37 +0800 Subject: [PATCH 33/45] chore: bump connector version --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 547a37b8e..1bfce0735 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ From 28fc2b9ee985cdce7c43114316a1fbcc80ab997d Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 1 Dec 2022 12:20:11 +0100 Subject: [PATCH 34/45] fix(mgmt_api): add missing error codes in schema --- apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index ed2aa0c6d..2cb5a4efe 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -62,7 +62,11 @@ schema("/subscriptions") -> tags => [<<"Subscriptions">>], parameters => parameters(), responses => #{ - 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{}) + 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{}), + 400 => emqx_dashboard_swagger:error_codes( + ['INVALID_PARAMETER'], <<"Invalid parameter">> + ), + 500 => emqx_dashboard_swagger:error_codes(['NODE_DOWN'], <<"Bad RPC">>) } } }. From 3bda905e92662d0cc13680e03294d98760fdc0e5 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 28 Nov 2022 11:27:44 +0100 Subject: [PATCH 35/45] refactor: return `204` instead of `200` for `PUT /authenticator/:id` --- apps/emqx_authn/src/emqx_authn_api.erl | 7 +++---- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 9 ++++----- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 2 +- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 2 ++ 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ee8757774..6627e935a 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1006,11 +1006,10 @@ update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> ) of {ok, #{ - post_config_update := #{emqx_authentication := #{id := ID}}, - raw_config := AuthenticatorsConfig + post_config_update := #{emqx_authentication := #{id := _ID}}, + raw_config := _AuthenticatorsConfig }} -> - {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), - {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; + {204}; {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> serialize_error(Reason); {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index a1047e25c..466aaddbe 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -22,7 +22,6 @@ -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). -define(TCP_DEFAULT, 'tcp:default'). @@ -42,7 +41,7 @@ groups() -> init_per_testcase(t_authenticator_fail, Config) -> meck:expect(emqx_authn_proto_v1, lookup_from_all_nodes, 3, [{error, {exception, badarg}}]), init_per_testcase(default, Config); -init_per_testcase(_, Config) -> +init_per_testcase(_Case, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [?CONF_NS_ATOM], @@ -213,7 +212,7 @@ test_authenticators(PathPrefix) -> method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>} }, - {ok, 200, _} = request( + {ok, 204, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig1 @@ -302,14 +301,14 @@ test_authenticator(PathPrefix) -> method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>} }, - {ok, 200, _} = request( + {ok, 204, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig1 ), ValidConfig2 = ValidConfig0#{pool_size => 9}, - {ok, 200, _} = request( + {ok, 204, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig2 diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index e84f87f0b..547f173d5 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -448,7 +448,7 @@ t_jwt_not_allow_empty_claim_name(_) -> ), ?assertMatch( - {200, _}, + {204}, emqx_authn_api:authenticator( put, #{ bindings => #{id => <<"jwt">>}, diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 0828616de..076ec6e69 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -14,6 +14,8 @@ - Refactor authn API by replacing `POST /authentication/{id}/move` with `PUT /authentication/{id}/position/{position}`. [#9419](https://github.com/emqx/emqx/pull/9419). Same is done for `/listeners/{listener_id}/authentication/id/...`. +- Return `204` instead of `200` for `PUT /authenticator/:id` [#9434](https://github.com/emqx/emqx/pull/9434/). + ## Bug fixes - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index bfe75ab36..3f4c52b75 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -4,6 +4,8 @@ - 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 +- 现在，`PUT /authenticator/:id` 将会返回 204 而不再是 200 [#9434](https://github.com/emqx/emqx/pull/9434/)。 + - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 - 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 From 9100b13db60099100f68f96c1b0bbc6e4aedb6c1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 2 Dec 2022 13:23:40 +0100 Subject: [PATCH 36/45] docs: fix schema for /trace/:name/log --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 4 ++-- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 587257688..466b0e82e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -172,8 +172,8 @@ schema("/trace/:name/log") -> responses => #{ 200 => [ - {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})} - | fields(bytes) ++ fields(position) + {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}, + {meta, fields(bytes) ++ fields(position)} ] } } diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 076ec6e69..0a52f6f08 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -19,3 +19,5 @@ ## Bug fixes - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). + +- Fix schema for `GET /trace/:name/log` [#9468](https://github.com/emqx/emqx/pull/9468). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 3f4c52b75..8c58cdec2 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -18,3 +18,5 @@ ## 修复 - 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 + +- [FIXME] Fix schema for `GET /trace/:name/log` [#9468](https://github.com/emqx/emqx/pull/9468). From f8dd8d652dc584cc3fbd223557efbb82f9691efb Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 2 Dec 2022 13:39:58 +0100 Subject: [PATCH 37/45] docs: add swagger error codes for /trace/:name/[log|download] --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 466b0e82e..52a8c1040 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -140,7 +140,8 @@ schema("/trace/:name/download") -> 'application/octet-stream' => #{schema => #{type => "string", format => "binary"}} } - } + }, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) } } }; @@ -174,7 +175,10 @@ schema("/trace/:name/log") -> [ {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}, {meta, fields(bytes) ++ fields(position)} - ] + ], + 400 => emqx_dashboard_swagger:error_codes( + ['READ_FILE_ERROR', 'RPC_ERROR', 'NODE_ERROR'], <<"Trace Log Failed">> + ) } } }. From 1ba66fc722260314daeb99cad3fa3cb22127ecf5 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 2 Dec 2022 14:04:26 +0100 Subject: [PATCH 38/45] docs: /trace returns list not single trace --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 52a8c1040..93d647753 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -72,7 +72,7 @@ schema("/trace") -> description => "List all trace", tags => ?TAGS, responses => #{ - 200 => hoconsc:ref(trace) + 200 => hoconsc:array(hoconsc:ref(trace)) } }, post => #{ From 3e842e4a6b0b2f60adb6569a9ee4f295ab03bb67 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 2 Dec 2022 13:42:57 +0100 Subject: [PATCH 39/45] style: fix changelog --- changes/v5.0.12-en.md | 2 +- changes/v5.0.12-zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 0a52f6f08..731719c7d 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -20,4 +20,4 @@ - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). -- Fix schema for `GET /trace/:name/log` [#9468](https://github.com/emqx/emqx/pull/9468). +- Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 8c58cdec2..4a7ad20b5 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -19,4 +19,4 @@ - 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 -- [FIXME] Fix schema for `GET /trace/:name/log` [#9468](https://github.com/emqx/emqx/pull/9468). +- [FIXME] Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). From 5820bae7be2eed8c1cb94b58895989eaf02959d9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 5 Dec 2022 10:37:15 +0100 Subject: [PATCH 40/45] chore: add zn translation --- changes/v5.0.12-zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 4a7ad20b5..e7cbd124c 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -19,4 +19,4 @@ - 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 -- [FIXME] Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). +- 修复 /trace API 的返回值格式和相关文档 [#9468](https://github.com/emqx/emqx/pull/9468)。 From 2443500884b766214ddc7fec418c82543d3af0f6 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 1 Dec 2022 16:42:31 +0100 Subject: [PATCH 41/45] fix: return 404 for /telemetry/data in case it's disabled --- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_modules/src/emqx_telemetry_api.erl | 25 ++++++++++++++---- .../test/emqx_telemetry_api_SUITE.erl | 26 ++++++++++++++++++- changes/v5.0.12-en.md | 2 ++ changes/v5.0.12-zh.md | 2 ++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 29a7d0362..fa68e17cb 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {modules, []}, {applications, [kernel, stdlib, emqx]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index a135ac3c3..4883de2fc 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -36,6 +36,7 @@ ]). -define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -63,7 +64,8 @@ schema("/telemetry/status") -> 'requestBody' => status_schema(?DESC(update_telemetry_status_api)), responses => #{ - 200 => status_schema(?DESC(update_telemetry_status_api)) + 200 => status_schema(?DESC(update_telemetry_status_api)), + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) } } }; @@ -75,7 +77,12 @@ schema("/telemetry/data") -> description => ?DESC(get_telemetry_data_api), tags => [<<"Telemetry">>], responses => - #{200 => mk(ref(?MODULE, telemetry), #{desc => ?DESC(get_telemetry_data_api)})} + #{ + 200 => mk(ref(?MODULE, telemetry), #{desc => ?DESC(get_telemetry_data_api)}), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Telemetry is not enabled">> + ) + } } }. @@ -220,21 +227,29 @@ status(put, #{body := Body}) -> true -> <<"Telemetry status is already enabled">>; false -> <<"Telemetry status is already disabled">> end, - {400, #{code => 'BAD_REQUEST', message => Reason}}; + {400, #{code => ?BAD_REQUEST, message => Reason}}; false -> case enable_telemetry(Enable) of ok -> {200, get_telemetry_status()}; {error, Reason} -> {400, #{ - code => 'BAD_REQUEST', + code => ?BAD_REQUEST, message => Reason }} end end. data(get, _Request) -> - {200, emqx_json:encode(get_telemetry_data())}. + case emqx_modules_conf:is_telemetry_enabled() of + true -> + {200, emqx_json:encode(get_telemetry_data())}; + false -> + {404, #{ + code => ?NOT_FOUND, + message => <<"Telemetry is not enabled">> + }} + end. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index d8e22cc1e..d2c8079b9 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -61,8 +61,20 @@ init_per_testcase(t_status_non_official, Config) -> init_per_testcase(t_status, Config) -> meck:new(emqx_telemetry, [non_strict, passthrough]), meck:expect(emqx_telemetry, enable, fun() -> ok end), + {ok, _, _} = + request( + put, + uri(["telemetry", "status"]), + #{<<"enable">> => true} + ), Config; init_per_testcase(_TestCase, Config) -> + {ok, _, _} = + request( + put, + uri(["telemetry", "status"]), + #{<<"enable">> => true} + ), Config. end_per_testcase(t_status_non_official, _Config) -> @@ -169,4 +181,16 @@ t_data(_) -> <<"vm_specs">> := _ }, jsx:decode(Result) - ). + ), + + {ok, 200, _} = + request( + put, + uri(["telemetry", "status"]), + #{<<"enable">> => false} + ), + + {ok, 404, _} = + request(get, uri(["telemetry", "data"])), + + ok. diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 731719c7d..0b6a6feed 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -21,3 +21,5 @@ - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). - Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). + +- Return `404` for `/telemetry/data` in case it's disabled [#9464](https://github.com/emqx/emqx/pull/9464). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index e7cbd124c..cde03f3d8 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -20,3 +20,5 @@ - 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 - 修复 /trace API 的返回值格式和相关文档 [#9468](https://github.com/emqx/emqx/pull/9468)。 + +- 在遥测功能未开启时，通过 /telemetry/data 请求其数据，将会返回 404 [#9464](https://github.com/emqx/emqx/pull/9464)。 From 0be5c8e84b4a15699c6461e082806923cd0660b0 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 5 Dec 2022 15:52:05 +0100 Subject: [PATCH 42/45] fix: Revert "refactor: return `204` instead of `200` for `PUT /authenticator/:id`" This reverts commit 3bda905e92662d0cc13680e03294d98760fdc0e5. --- apps/emqx_authn/src/emqx_authn_api.erl | 7 ++++--- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 9 +++++---- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 2 +- changes/v5.0.12-en.md | 2 -- changes/v5.0.12-zh.md | 2 -- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 6627e935a..ee8757774 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1006,10 +1006,11 @@ update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> ) of {ok, #{ - post_config_update := #{emqx_authentication := #{id := _ID}}, - raw_config := _AuthenticatorsConfig + post_config_update := #{emqx_authentication := #{id := ID}}, + raw_config := AuthenticatorsConfig }} -> - {204}; + {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), + {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> serialize_error(Reason); {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 466aaddbe..a1047e25c 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -22,6 +22,7 @@ -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(TCP_DEFAULT, 'tcp:default'). @@ -41,7 +42,7 @@ groups() -> init_per_testcase(t_authenticator_fail, Config) -> meck:expect(emqx_authn_proto_v1, lookup_from_all_nodes, 3, [{error, {exception, badarg}}]), init_per_testcase(default, Config); -init_per_testcase(_Case, Config) -> +init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [?CONF_NS_ATOM], @@ -212,7 +213,7 @@ test_authenticators(PathPrefix) -> method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>} }, - {ok, 204, _} = request( + {ok, 200, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig1 @@ -301,14 +302,14 @@ test_authenticator(PathPrefix) -> method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>} }, - {ok, 204, _} = request( + {ok, 200, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig1 ), ValidConfig2 = ValidConfig0#{pool_size => 9}, - {ok, 204, _} = request( + {ok, 200, _} = request( put, uri(PathPrefix ++ [?CONF_NS, "password_based:http"]), ValidConfig2 diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 547f173d5..e84f87f0b 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -448,7 +448,7 @@ t_jwt_not_allow_empty_claim_name(_) -> ), ?assertMatch( - {204}, + {200, _}, emqx_authn_api:authenticator( put, #{ bindings => #{id => <<"jwt">>}, diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 0b6a6feed..9baaa9d75 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -14,8 +14,6 @@ - Refactor authn API by replacing `POST /authentication/{id}/move` with `PUT /authentication/{id}/position/{position}`. [#9419](https://github.com/emqx/emqx/pull/9419). Same is done for `/listeners/{listener_id}/authentication/id/...`. -- Return `204` instead of `200` for `PUT /authenticator/:id` [#9434](https://github.com/emqx/emqx/pull/9434/). - ## Bug fixes - Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index cde03f3d8..82c2796c4 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -4,8 +4,6 @@ - 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 -- 现在，`PUT /authenticator/:id` 将会返回 204 而不再是 200 [#9434](https://github.com/emqx/emqx/pull/9434/)。 - - 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 - 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 From 72669080a15c3f5fff943ab7186d25274373b684 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 5 Dec 2022 23:16:07 +0800 Subject: [PATCH 43/45] fix(frame): fix potential parse errors found by fuzzing test --- apps/emqx/src/emqx_frame.erl | 91 +++++++++++++-------- apps/emqx/test/emqx_frame_SUITE.erl | 12 +++ apps/emqx/test/emqx_packet_SUITE.erl | 6 +- apps/emqx/test/emqx_ws_connection_SUITE.erl | 2 +- changes/v5.0.12-en.md | 2 + changes/v5.0.12-zh.md | 2 + 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 30b2fb618..be576105f 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -182,8 +182,19 @@ parse_remaining_len( Packet = packet(Header, #mqtt_packet_disconnect{reason_code = ?RC_SUCCESS}), {ok, Packet, Rest, ?NONE(Options)}; %% Match PINGREQ. -parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, Options) -> +parse_remaining_len( + <<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?PINGREQ}, 1, 0, Options +) -> parse_frame(Rest, Header, 0, Options); +parse_remaining_len( + <<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?PINGRESP}, 1, 0, Options +) -> + parse_frame(Rest, Header, 0, Options); +%% Without this clause a crash may will happen when data incorrect, this was found by fuzzing test +parse_remaining_len( + <<0:8, _Rest/binary>>, _Header, 1, 0, _Options +) -> + ?PARSE_ERR(invalid_remaining_len); %% Match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK... parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) -> parse_frame(Rest, Header, 2, Options); @@ -261,41 +272,51 @@ parse_packet( #{strict_mode := StrictMode} ) -> {ProtoName, Rest} = parse_utf8_string(FrameBin, StrictMode), - <> = Rest, - % Note: Crash when reserved flag doesn't equal to 0, there is no strict - % compliance with the MQTT5.0. - <> = Rest1, - - {Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode), - {ClientId, Rest4} = parse_utf8_string(Rest3, StrictMode), - ConnPacket = #mqtt_packet_connect{ - proto_name = ProtoName, - proto_ver = ProtoVer, - is_bridge = (BridgeTag =:= 8), - clean_start = bool(CleanStart), - will_flag = bool(WillFlag), - will_qos = WillQoS, - will_retain = bool(WillRetain), - keepalive = KeepAlive, - properties = Properties, - clientid = ClientId - }, - {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4, StrictMode), - {Username, Rest6} = parse_utf8_string(Rest5, StrictMode, bool(UsernameFlag)), - {Password, <<>>} = parse_utf8_string(Rest6, StrictMode, bool(PasswordFlag)), - ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; + case Rest of + % Note: Crash when reserved flag doesn't equal to 0, there is no strict + % compliance with the MQTT5.0. + <> -> + {Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode), + {ClientId, Rest4} = parse_utf8_string(Rest3, StrictMode), + ConnPacket = #mqtt_packet_connect{ + proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = (BridgeTag =:= 8), + clean_start = bool(CleanStart), + will_flag = bool(WillFlag), + will_qos = WillQoS, + will_retain = bool(WillRetain), + keepalive = KeepAlive, + properties = Properties, + clientid = ClientId + }, + {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4, StrictMode), + {Username, Rest6} = parse_utf8_string(Rest5, StrictMode, bool(UsernameFlag)), + case parse_utf8_string(Rest6, StrictMode, bool(PasswordFlag)) of + {Password, <<>>} -> + ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; + _ -> + ?PARSE_ERR(malformed_connect_payload) + end; + _ -> + ?PARSE_ERR(malformed_connect_header) + end; parse_packet( #mqtt_packet_header{type = ?CONNACK}, <>, #{version := Ver, strict_mode := StrictMode} ) -> - {Properties, <<>>} = parse_properties(Rest, Ver, StrictMode), - #mqtt_packet_connack{ - ack_flags = AckFlags, - reason_code = ReasonCode, - properties = Properties - }; + case parse_properties(Rest, Ver, StrictMode) of + {Properties, <<>>} -> + #mqtt_packet_connack{ + ack_flags = AckFlags, + reason_code = ReasonCode, + properties = Properties + }; + _ -> + ?PARSE_ERR(malformed_properties) + end; parse_packet( #mqtt_packet_header{type = ?PUBLISH, qos = QoS}, Bin, @@ -411,7 +432,9 @@ parse_packet( #{strict_mode := StrictMode, version := ?MQTT_PROTO_V5} ) -> {Properties, <<>>} = parse_properties(Rest, ?MQTT_PROTO_V5, StrictMode), - #mqtt_packet_auth{reason_code = ReasonCode, properties = Properties}. + #mqtt_packet_auth{reason_code = ReasonCode, properties = Properties}; +parse_packet(_Header, _FrameBin, _Options) -> + ?PARSE_ERR(malformed_packet). parse_will_message( Packet = #mqtt_packet_connect{ @@ -437,7 +460,9 @@ parse_will_message(Packet, Bin, _StrictMode) -> -compile({inline, [parse_packet_id/1]}). parse_packet_id(<>) -> - {PacketId, Rest}. + {PacketId, Rest}; +parse_packet_id(_) -> + ?PARSE_ERR(invalid_packet_id). parse_properties(Bin, Ver, _StrictMode) when Ver =/= ?MQTT_PROTO_V5 -> {#{}, Bin}; diff --git a/apps/emqx/test/emqx_frame_SUITE.erl b/apps/emqx/test/emqx_frame_SUITE.erl index 906b976ac..ee81655bf 100644 --- a/apps/emqx/test/emqx_frame_SUITE.erl +++ b/apps/emqx/test/emqx_frame_SUITE.erl @@ -619,6 +619,18 @@ t_serialize_parse_auth_v5(_) -> }) ). +t_parse_invalid_remaining_len(_) -> + ?assertException( + throw, {frame_parse_error, invalid_remaining_len}, emqx_frame:parse(<>) + ). + +t_parse_malformed_properties(_) -> + ?assertException( + throw, + {frame_parse_error, malformed_properties}, + emqx_frame:parse(<<2:4, 0:4, 3:8, 1:8, 0:8, 0:8>>) + ). + parse_serialize(Packet) -> parse_serialize(Packet, #{strict_mode => true}). diff --git a/apps/emqx/test/emqx_packet_SUITE.erl b/apps/emqx/test/emqx_packet_SUITE.erl index ff50c3ebe..ba38c1f08 100644 --- a/apps/emqx/test/emqx_packet_SUITE.erl +++ b/apps/emqx/test/emqx_packet_SUITE.erl @@ -212,7 +212,9 @@ t_check_publish(_) -> ?PUBLISH_PACKET(1, <<"topic">>, 1, #{'Topic-Alias' => 0}, <<"payload">>) ), %% TODO:: - %% {error, ?RC_PROTOCOL_ERROR} = emqx_packet:check(?PUBLISH_PACKET(1, <<"topic">>, 1, #{'Subscription-Identifier' => 10}, <<"payload">>)), + %% {error, ?RC_PROTOCOL_ERROR} = emqx_packet:check( + %% ?PUBLISH_PACKET(1, <<"topic">>, 1, #{'Subscription-Identifier' => 10}, <<"payload">>) + %%), ok = emqx_packet:check( ?PUBLISH_PACKET(1, <<"topic">>, 1, #{'Subscription-Identifier' => 10}, <<"payload">>) ), @@ -414,5 +416,5 @@ t_format(_) -> t_parse_empty_publish(_) -> %% 52: 0011(type=PUBLISH) 0100 (QoS=2) - {ok, Packet, <<>>, {none, _}} = emqx_frame:parse(<<52, 0>>), + Packet = #mqtt_packet_publish{topic_name = <<>>}, ?assertEqual({error, ?RC_PROTOCOL_ERROR}, emqx_packet:check(Packet)). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 47efc1829..bb0a6cb05 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -535,7 +535,7 @@ t_parse_incoming(_) -> t_parse_incoming_frame_error(_) -> {Packets, _St} = ?ws_conn:parse_incoming(<<3, 2, 1, 0>>, [], st()), - FrameError = {frame_error, function_clause}, + FrameError = {frame_error, malformed_packet}, [{incoming, FrameError}] = Packets. t_handle_incomming_frame_error(_) -> diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 0b6a6feed..e5dfc9dd6 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -23,3 +23,5 @@ - Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). - Return `404` for `/telemetry/data` in case it's disabled [#9464](https://github.com/emqx/emqx/pull/9464). + +- Fix some potential MQTT packet parse errors [#9477](https://github.com/emqx/emqx/pull/9477). diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index cde03f3d8..593babb75 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -22,3 +22,5 @@ - 修复 /trace API 的返回值格式和相关文档 [#9468](https://github.com/emqx/emqx/pull/9468)。 - 在遥测功能未开启时，通过 /telemetry/data 请求其数据，将会返回 404 [#9464](https://github.com/emqx/emqx/pull/9464)。 + +- 修复了一些 MQTT 协议包的潜在解析错误 [#9477](https://github.com/emqx/emqx/pull/9477)。 From 817554c7d756f2d4c955328ecab1e3274128bee1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 2 Dec 2022 11:38:08 +0100 Subject: [PATCH 44/45] refactor(emqx_frame): better error handling --- apps/emqx/src/emqx_frame.erl | 141 ++++++++++++++++++---------- apps/emqx/test/emqx_frame_SUITE.erl | 23 ++++- changes/v5.0.12-en.md.orig | 29 ++++++ changes/v5.0.12-zh.md.orig | 28 ++++++ 4 files changed, 169 insertions(+), 52 deletions(-) create mode 100644 changes/v5.0.12-en.md.orig create mode 100644 changes/v5.0.12-zh.md.orig diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index be576105f..a86bb0155 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -187,14 +187,14 @@ parse_remaining_len( ) -> parse_frame(Rest, Header, 0, Options); parse_remaining_len( - <<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?PINGRESP}, 1, 0, Options + <<0:8, _Rest/binary>>, _Header = #mqtt_packet_header{type = ?PINGRESP}, 1, 0, _Options ) -> - parse_frame(Rest, Header, 0, Options); -%% Without this clause a crash may will happen when data incorrect, this was found by fuzzing test + ?PARSE_ERR(#{hint => unexpected_packet, header_type => 'PINGRESP'}); +%% All other types of messages should not have a zero remaining length. parse_remaining_len( - <<0:8, _Rest/binary>>, _Header, 1, 0, _Options + <<0:8, _Rest/binary>>, Header, 1, 0, _Options ) -> - ?PARSE_ERR(invalid_remaining_len); + ?PARSE_ERR(#{hint => zero_remaining_len, header_type => Header#mqtt_packet_header.type}); %% Match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK... parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) -> parse_frame(Rest, Header, 2, Options); @@ -266,42 +266,75 @@ packet(Header, Variable) -> packet(Header, Variable, Payload) -> #mqtt_packet{header = Header, variable = Variable, payload = Payload}. +parse_connect(FrameBin, StrictMode) -> + {ProtoName, Rest} = parse_utf8_string_with_hint(FrameBin, StrictMode, invalid_proto_name), + case ProtoName of + <<"MQTT">> -> + ok; + <<"MQIsdp">> -> + ok; + _ -> + %% from spec: the server MAY send disconnect with reason code 0x84 + %% we chose to close socket because the client is likely not talking MQTT anyway + ?PARSE_ERR(#{ + hint => invalid_proto_name, + expected => <<"'MQTT' or 'MQIsdp'">>, + received => ProtoName + }) + end, + parse_connect2(ProtoName, Rest, StrictMode). + +% Note: return malformed if reserved flag is not 0. +parse_connect2( + ProtoName, + <>, + StrictMode +) -> + {Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode), + {ClientId, Rest4} = parse_utf8_string_with_hint(Rest3, StrictMode, invalid_username), + ConnPacket = #mqtt_packet_connect{ + proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = (BridgeTag =:= 8), + clean_start = bool(CleanStart), + will_flag = bool(WillFlag), + will_qos = WillQoS, + will_retain = bool(WillRetain), + keepalive = KeepAlive, + properties = Properties, + clientid = ClientId + }, + {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4, StrictMode), + {Username, Rest6} = parse_optional( + Rest5, + fun(Bin) -> + parse_utf8_string_with_hint(Bin, StrictMode, invalid_username) + end, + bool(UsernameFlag) + ), + {Password, Rest7} = parse_optional( + Rest6, + fun(Bin) -> + parse_utf8_string_with_hint(Bin, StrictMode, invalid_password) + end, + bool(PasswordFlag) + ), + case Rest7 of + <<>> -> + ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; + _ -> + ?PARSE_ERR(malformed_connect_payload) + end; +parse_connect2(_ProtoName, _, _) -> + ?PARSE_ERR(malformed_connect_header). + parse_packet( #mqtt_packet_header{type = ?CONNECT}, FrameBin, #{strict_mode := StrictMode} ) -> - {ProtoName, Rest} = parse_utf8_string(FrameBin, StrictMode), - case Rest of - % Note: Crash when reserved flag doesn't equal to 0, there is no strict - % compliance with the MQTT5.0. - <> -> - {Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode), - {ClientId, Rest4} = parse_utf8_string(Rest3, StrictMode), - ConnPacket = #mqtt_packet_connect{ - proto_name = ProtoName, - proto_ver = ProtoVer, - is_bridge = (BridgeTag =:= 8), - clean_start = bool(CleanStart), - will_flag = bool(WillFlag), - will_qos = WillQoS, - will_retain = bool(WillRetain), - keepalive = KeepAlive, - properties = Properties, - clientid = ClientId - }, - {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4, StrictMode), - {Username, Rest6} = parse_utf8_string(Rest5, StrictMode, bool(UsernameFlag)), - case parse_utf8_string(Rest6, StrictMode, bool(PasswordFlag)) of - {Password, <<>>} -> - ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; - _ -> - ?PARSE_ERR(malformed_connect_payload) - end; - _ -> - ?PARSE_ERR(malformed_connect_header) - end; + parse_connect(FrameBin, StrictMode); parse_packet( #mqtt_packet_header{type = ?CONNACK}, <>, @@ -322,7 +355,7 @@ parse_packet( Bin, #{strict_mode := StrictMode, version := Ver} ) -> - {TopicName, Rest} = parse_utf8_string(Bin, StrictMode), + {TopicName, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_topic), {PacketId, Rest1} = case QoS of ?QOS_0 -> {undefined, Rest}; @@ -445,7 +478,7 @@ parse_will_message( StrictMode ) -> {Props, Rest} = parse_properties(Bin, Ver, StrictMode), - {Topic, Rest1} = parse_utf8_string(Rest, StrictMode), + {Topic, Rest1} = parse_utf8_string_with_hint(Rest, StrictMode, invalid_topic), {Payload, Rest2} = parse_binary_data(Rest1), { Packet#mqtt_packet_connect{ @@ -483,10 +516,10 @@ parse_property(<<16#01, Val, Bin/binary>>, Props, StrictMode) -> parse_property(<<16#02, Val:32/big, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Message-Expiry-Interval' => Val}, StrictMode); parse_property(<<16#03, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_content_type), parse_property(Rest, Props#{'Content-Type' => Val}, StrictMode); parse_property(<<16#08, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_topic), parse_property(Rest, Props#{'Response-Topic' => Val}, StrictMode); parse_property(<<16#09, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Correlation-Data' => Val}, StrictMode); @@ -496,12 +529,12 @@ parse_property(<<16#0B, Bin/binary>>, Props, StrictMode) -> parse_property(<<16#11, Val:32/big, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Session-Expiry-Interval' => Val}, StrictMode); parse_property(<<16#12, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_assigned_client_id), parse_property(Rest, Props#{'Assigned-Client-Identifier' => Val}, StrictMode); parse_property(<<16#13, Val:16, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Server-Keep-Alive' => Val}, StrictMode); parse_property(<<16#15, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_authn_method), parse_property(Rest, Props#{'Authentication-Method' => Val}, StrictMode); parse_property(<<16#16, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Authentication-Data' => Val}, StrictMode); @@ -512,13 +545,13 @@ parse_property(<<16#18, Val:32, Bin/binary>>, Props, StrictMode) -> parse_property(<<16#19, Val, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Request-Response-Information' => Val}, StrictMode); parse_property(<<16#1A, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_info), parse_property(Rest, Props#{'Response-Information' => Val}, StrictMode); parse_property(<<16#1C, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_server_reference), parse_property(Rest, Props#{'Server-Reference' => Val}, StrictMode); parse_property(<<16#1F, Bin/binary>>, Props, StrictMode) -> - {Val, Rest} = parse_utf8_string(Bin, StrictMode), + {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_reason_string), parse_property(Rest, Props#{'Reason-String' => Val}, StrictMode); parse_property(<<16#21, Val:16/big, Bin/binary>>, Props, StrictMode) -> parse_property(Bin, Props#{'Receive-Maximum' => Val}, StrictMode); @@ -609,10 +642,18 @@ parse_utf8_pair(Bin, _StrictMode) when total_bytes => byte_size(Bin) }). -parse_utf8_string(Bin, _StrictMode, false) -> - {undefined, Bin}; -parse_utf8_string(Bin, StrictMode, true) -> - parse_utf8_string(Bin, StrictMode). +parse_utf8_string_with_hint(Bin, StrictMode, Hint) -> + try + parse_utf8_string(Bin, StrictMode) + catch + throw:{?FRAME_PARSE_ERROR, Reason} when is_map(Reason) -> + ?PARSE_ERR(Reason#{hint => Hint}) + end. + +parse_optional(Bin, F, true) -> + F(Bin); +parse_optional(Bin, _F, false) -> + {undefined, Bin}. parse_utf8_string(<>, true) -> {validate_utf8(Str), Rest}; @@ -629,7 +670,7 @@ parse_utf8_string(<>, _) when parse_utf8_string(Bin, _) when 2 > byte_size(Bin) -> - ?PARSE_ERR(malformed_utf8_string_length). + ?PARSE_ERR(#{reason => malformed_utf8_string_length}). parse_binary_data(<>) -> {Data, Rest}; diff --git a/apps/emqx/test/emqx_frame_SUITE.erl b/apps/emqx/test/emqx_frame_SUITE.erl index ee81655bf..f6296359f 100644 --- a/apps/emqx/test/emqx_frame_SUITE.erl +++ b/apps/emqx/test/emqx_frame_SUITE.erl @@ -576,7 +576,12 @@ t_serialize_parse_pingreq(_) -> t_serialize_parse_pingresp(_) -> PingResp = ?PACKET(?PINGRESP), - ?assertEqual(PingResp, parse_serialize(PingResp)). + Packet = serialize_to_binary(PingResp), + ?assertException( + throw, + {frame_parse_error, #{hint := unexpected_packet, header_type := 'PINGRESP'}}, + emqx_frame:parse(Packet) + ). t_parse_disconnect(_) -> Packet = ?DISCONNECT_PACKET(?RC_SUCCESS), @@ -621,7 +626,7 @@ t_serialize_parse_auth_v5(_) -> t_parse_invalid_remaining_len(_) -> ?assertException( - throw, {frame_parse_error, invalid_remaining_len}, emqx_frame:parse(<>) + throw, {frame_parse_error, #{hint := zero_remaining_len}}, emqx_frame:parse(<>) ). t_parse_malformed_properties(_) -> @@ -631,6 +636,20 @@ t_parse_malformed_properties(_) -> emqx_frame:parse(<<2:4, 0:4, 3:8, 1:8, 0:8, 0:8>>) ). +t_parse_malformed_connect(_) -> + ?assertException( + throw, + {frame_parse_error, malformed_connect_header}, + emqx_frame:parse(<<16, 11, 0, 6, 77, 81, 73, 115, 110, 112, 3, 130, 1, 6>>) + ), + ?assertException( + throw, + {frame_parse_error, malformed_connect_payload}, + emqx_frame:parse( + <<16, 21, 0, 6, 77, 81, 73, 115, 110, 112, 3, 130, 1, 6, 0, 0, 2, 67, 49.49>> + ) + ). + parse_serialize(Packet) -> parse_serialize(Packet, #{strict_mode => true}). diff --git a/changes/v5.0.12-en.md.orig b/changes/v5.0.12-en.md.orig new file mode 100644 index 000000000..799c1350a --- /dev/null +++ b/changes/v5.0.12-en.md.orig @@ -0,0 +1,29 @@ +# v5.0.12 + +## Enhancements + +- Disable global garbage collection by `node.global_gc_interval = disabled` [#9418](https://github.com/emqx/emqx/pull/9418)。 + +- Improve the CLI to avoid waste atom table when typing erros [#9416](https://github.com/emqx/emqx/pull/9416). + +- Start building MacOS packages for Apple Silicon hadrdware [#9423](https://github.com/emqx/emqx/pull/9423). + +- Remove support for setting shared subscriptions using the non-standard `$queue` feature [#9412](https://github.com/emqx/emqx/pull/9412). + Shared subscriptions are now part of the MQTT spec. Use `$share` instead. + +- Refactor authn API by replacing `POST /authentication/{id}/move` with `PUT /authentication/{id}/position/{position}`. [#9419](https://github.com/emqx/emqx/pull/9419). + Same is done for `/listeners/{listener_id}/authentication/id/...`. + +- Return `204` instead of `200` for `PUT /authenticator/:id` [#9434](https://github.com/emqx/emqx/pull/9434/). + +## Bug fixes + +- Fix that the obsolete SSL files aren't deleted after the ExHook config update [#9432](https://github.com/emqx/emqx/pull/9432). + +- Fix doc and schema for `/trace` API [#9468](https://github.com/emqx/emqx/pull/9468). + +<<<<<<< HEAD +- Return `404` for `/telemetry/data` in case it's disabled [#9464](https://github.com/emqx/emqx/pull/9464). +======= +- Fix some potential MQTT packet parse errors [#9477](https://github.com/emqx/emqx/pull/9477). +>>>>>>> 030a07d8e (fix(frame): fix potential parse errors found by fuzzing test) diff --git a/changes/v5.0.12-zh.md.orig b/changes/v5.0.12-zh.md.orig new file mode 100644 index 000000000..c9da9ec7d --- /dev/null +++ b/changes/v5.0.12-zh.md.orig @@ -0,0 +1,28 @@ +# v5.0.12 + +## 增强 + +- 通过 `node.global_gc_interval = disabled` 来禁用全局垃圾回收 [#9418](https://github.com/emqx/emqx/pull/9418)。 + +- 现在，`PUT /authenticator/:id` 将会返回 204 而不再是 200 [#9434](https://github.com/emqx/emqx/pull/9434/)。 + +- 优化命令行实现, 避免输入错误指令时, 产生不必要的原子表消耗 [#9416](https://github.com/emqx/emqx/pull/9416)。 + +- 支持在 Apple Silicon 架构下编译苹果系统的发行版本 [#9423](https://github.com/emqx/emqx/pull/9423)。 + +- 删除了老的共享订阅支持方式， 不再使用 `$queue` 前缀 [#9412](https://github.com/emqx/emqx/pull/9412)。 + 共享订阅自 MQTT v5.0 开始已成为协议标准，可以使用 `$share` 前缀代替 `$queue`。 + +- 重构认证 API，使用 `PUT /authentication/{id}/position/{position}` 代替了 `POST /authentication/{id}/move` [#9419](https://github.com/emqx/emqx/pull/9419)。 + +## 修复 + +- 修复 ExHook 更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9432](https://github.com/emqx/emqx/pull/9432)。 + +- 修复 /trace API 的返回值格式和相关文档 [#9468](https://github.com/emqx/emqx/pull/9468)。 + +<<<<<<< HEAD +- 在遥测功能未开启时，通过 /telemetry/data 请求其数据，将会返回 404 [#9464](https://github.com/emqx/emqx/pull/9464)。 +======= +- 修复了一些 MQTT 协议包的潜在解析错误 [#9477](https://github.com/emqx/emqx/pull/9477)。 +>>>>>>> 030a07d8e (fix(frame): fix potential parse errors found by fuzzing test) From cca3421308b141427db5d4725d7c79e1a4a7bb8e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Nov 2022 19:40:40 +0100 Subject: [PATCH 45/45] refactor: use static function references --- .../src/emqx_rule_api_schema.erl | 122 +++++++----------- .../src/emqx_rule_engine.app.src | 2 +- .../emqx_rule_engine/src/emqx_rule_events.erl | 78 +++++++---- .../test/emqx_rule_events_SUITE.erl | 46 +++++-- 4 files changed, 136 insertions(+), 112 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 5c1ef9cd0..edfce5fa7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -65,10 +65,7 @@ fields("rule_info") -> ] ++ fields("rule_creation"); %% TODO: we can delete this API if the Dashboard not depends on it fields("rule_events") -> - ETopics = [ - binary_to_atom(emqx_rule_events:event_topic(E)) - || E <- emqx_rule_events:event_names() - ], + ETopics = emqx_rule_events:event_topics_enum(), [ {"event", sc(hoconsc:enum(ETopics), #{desc => ?DESC("rs_event"), required => true})}, {"title", sc(binary(), #{desc => ?DESC("rs_title"), example => "some title"})}, @@ -150,77 +147,43 @@ fields("node_metrics") -> fields("metrics"); fields("ctx_pub") -> [ - {"event_type", sc(message_publish, #{desc => ?DESC("event_event_type"), required => true})}, - {"id", sc(binary(), #{desc => ?DESC("event_id")})}, - {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, - {"username", sc(binary(), #{desc => ?DESC("event_username")})}, - {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, - {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, - {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, - {"publish_received_at", - sc(integer(), #{ - desc => ?DESC("event_publish_received_at") - })} - ] ++ [qos()]; + {"event_type", event_type_sc(message_publish)}, + {"id", sc(binary(), #{desc => ?DESC("event_id")})} + | msg_event_common_fields() + ]; fields("ctx_sub") -> [ - {"event_type", - sc(session_subscribed, #{desc => ?DESC("event_event_type"), required => true})}, - {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, - {"username", sc(binary(), #{desc => ?DESC("event_username")})}, - {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, - {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, - {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, - {"publish_received_at", - sc(integer(), #{ - desc => ?DESC("event_publish_received_at") - })} - ] ++ [qos()]; + {"event_type", event_type_sc(session_subscribed)} + | msg_event_common_fields() + ]; fields("ctx_unsub") -> [ - {"event_type", - sc(session_unsubscribed, #{desc => ?DESC("event_event_type"), required => true})} - ] ++ - proplists:delete("event_type", fields("ctx_sub")); + {"event_type", event_type_sc(session_unsubscribed)} + | proplists:delete("event_type", fields("ctx_sub")) + ]; fields("ctx_delivered") -> [ - {"event_type", - sc(message_delivered, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(message_delivered)}, {"id", sc(binary(), #{desc => ?DESC("event_id")})}, {"from_clientid", sc(binary(), #{desc => ?DESC("event_from_clientid")})}, - {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})}, - {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, - {"username", sc(binary(), #{desc => ?DESC("event_username")})}, - {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, - {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, - {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, - {"publish_received_at", - sc(integer(), #{ - desc => ?DESC("event_publish_received_at") - })} - ] ++ [qos()]; + {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})} + | msg_event_common_fields() + ]; fields("ctx_acked") -> - [{"event_type", sc(message_acked, #{desc => ?DESC("event_event_type"), required => true})}] ++ - proplists:delete("event_type", fields("ctx_delivered")); + [ + {"event_type", event_type_sc(message_acked)} + | proplists:delete("event_type", fields("ctx_delivered")) + ]; fields("ctx_dropped") -> [ - {"event_type", sc(message_dropped, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(message_dropped)}, {"id", sc(binary(), #{desc => ?DESC("event_id")})}, - {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})}, - {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, - {"username", sc(binary(), #{desc => ?DESC("event_username")})}, - {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, - {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, - {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, - {"publish_received_at", - sc(integer(), #{ - desc => ?DESC("event_publish_received_at") - })} - ] ++ [qos()]; + {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})} + | msg_event_common_fields() + ]; fields("ctx_connected") -> [ - {"event_type", - sc(client_connected, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(client_connected)}, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, {"username", sc(binary(), #{desc => ?DESC("event_username")})}, {"mountpoint", sc(binary(), #{desc => ?DESC("event_mountpoint")})}, @@ -239,8 +202,7 @@ fields("ctx_connected") -> ]; fields("ctx_disconnected") -> [ - {"event_type", - sc(client_disconnected, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(client_disconnected)}, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, {"username", sc(binary(), #{desc => ?DESC("event_username")})}, {"reason", sc(binary(), #{desc => ?DESC("event_ctx_disconnected_reason")})}, @@ -253,7 +215,7 @@ fields("ctx_disconnected") -> ]; fields("ctx_connack") -> [ - {"event_type", sc(client_connack, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(client_connack)}, {"reason_code", sc(binary(), #{desc => ?DESC("event_ctx_connack_reason_code")})}, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, {"clean_start", sc(boolean(), #{desc => ?DESC("event_clean_start"), default => true})}, @@ -271,8 +233,7 @@ fields("ctx_connack") -> ]; fields("ctx_check_authz_complete") -> [ - {"event_type", - sc(client_check_authz_complete, #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc(client_check_authz_complete)}, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, {"username", sc(binary(), #{desc => ?DESC("event_username")})}, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, @@ -283,19 +244,16 @@ fields("ctx_check_authz_complete") -> ]; fields("ctx_bridge_mqtt") -> [ - {"event_type", - sc('$bridges/mqtt:*', #{desc => ?DESC("event_event_type"), required => true})}, + {"event_type", event_type_sc('$bridges/mqtt:*')}, {"id", sc(binary(), #{desc => ?DESC("event_id")})}, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, {"server", sc(binary(), #{desc => ?DESC("event_server")})}, {"dup", sc(binary(), #{desc => ?DESC("event_dup")})}, {"retain", sc(binary(), #{desc => ?DESC("event_retain")})}, - {"message_received_at", - sc(integer(), #{ - desc => ?DESC("event_publish_received_at") - })} - ] ++ [qos()]. + {"message_received_at", publish_received_at_sc()}, + qos() + ]. qos() -> {"qos", sc(emqx_schema:qos(), #{desc => ?DESC("event_qos")})}. @@ -312,4 +270,22 @@ rule_id() -> )}. sc(Type, Meta) -> hoconsc:mk(Type, Meta). + ref(Field) -> hoconsc:ref(?MODULE, Field). + +event_type_sc(Event) -> + sc(Event, #{desc => ?DESC("event_event_type"), required => true}). + +publish_received_at_sc() -> + sc(integer(), #{desc => ?DESC("event_publish_received_at")}). + +msg_event_common_fields() -> + [ + {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})}, + {"username", sc(binary(), #{desc => ?DESC("event_username")})}, + {"payload", sc(binary(), #{desc => ?DESC("event_payload")})}, + {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})}, + {"topic", sc(binary(), #{desc => ?DESC("event_topic")})}, + {"publish_received_at", publish_received_at_sc()}, + qos() + ]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 6419e4184..8608cd67a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index b80b9777a..b8e227a90 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -28,6 +28,7 @@ unload/1, event_names/0, event_name/1, + event_topics_enum/0, event_topic/1, eventmsg_publish/1 ]). @@ -78,6 +79,22 @@ event_names() -> 'delivery.dropped' ]. +%% for documentation purposes +event_topics_enum() -> + [ + '$events/client_connected', + '$events/client_disconnected', + '$events/client_connack', + '$events/client_check_authz_complete', + '$events/session_subscribed', + '$events/session_unsubscribed', + '$events/message_delivered', + '$events/message_acked', + '$events/message_dropped', + '$events/delivery_dropped' + % '$events/message_publish' % not possible to use in SELECT FROM + ]. + reload() -> lists:foreach( fun(Rule) -> @@ -88,21 +105,22 @@ reload() -> load(Topic) -> HookPoint = event_name(Topic), + HookFun = hook_fun_name(HookPoint), emqx_hooks:put( - HookPoint, {?MODULE, hook_fun(HookPoint), [#{event_topic => Topic}]}, ?HP_RULE_ENGINE + HookPoint, {?MODULE, HookFun, [#{event_topic => Topic}]}, ?HP_RULE_ENGINE ). unload() -> lists:foreach( fun(HookPoint) -> - emqx_hooks:del(HookPoint, {?MODULE, hook_fun(HookPoint)}) + emqx_hooks:del(HookPoint, {?MODULE, hook_fun_name(HookPoint)}) end, event_names() ). unload(Topic) -> HookPoint = event_name(Topic), - emqx_hooks:del(HookPoint, {?MODULE, hook_fun(HookPoint)}). + emqx_hooks:del(HookPoint, {?MODULE, hook_fun_name(HookPoint)}). %%-------------------------------------------------------------------- %% Callbacks @@ -987,15 +1005,25 @@ columns_example_props_specific(unsub_props) -> %% Helper functions %%-------------------------------------------------------------------- -hook_fun(<<"$bridges/", _/binary>>) -> - on_bridge_message_received; -hook_fun(Event) -> - case string:split(atom_to_list(Event), ".") of - [Prefix, Name] -> - list_to_atom(lists:append(["on_", Prefix, "_", Name])); - [_] -> - error(invalid_event, Event) - end. +hook_fun_name(HookPoint) -> + HookFun = hook_fun(HookPoint), + {name, HookFunName} = erlang:fun_info(HookFun, name), + HookFunName. + +%% return static function references to help static code checks +hook_fun(<<"$bridges/", _/binary>>) -> fun ?MODULE:on_bridge_message_received/2; +hook_fun('client.connected') -> fun ?MODULE:on_client_connected/3; +hook_fun('client.disconnected') -> fun ?MODULE:on_client_disconnected/4; +hook_fun('client.connack') -> fun ?MODULE:on_client_connack/4; +hook_fun('client.check_authz_complete') -> fun ?MODULE:on_client_check_authz_complete/6; +hook_fun('session.subscribed') -> fun ?MODULE:on_session_subscribed/4; +hook_fun('session.unsubscribed') -> fun ?MODULE:on_session_unsubscribed/4; +hook_fun('message.delivered') -> fun ?MODULE:on_message_delivered/3; +hook_fun('message.acked') -> fun ?MODULE:on_message_acked/3; +hook_fun('message.dropped') -> fun ?MODULE:on_message_dropped/4; +hook_fun('delivery.dropped') -> fun ?MODULE:on_delivery_dropped/4; +hook_fun('message.publish') -> fun ?MODULE:on_message_publish/2; +hook_fun(Event) -> error({invalid_event, Event}). reason(Reason) when is_atom(Reason) -> Reason; reason({shutdown, Reason}) when is_atom(Reason) -> Reason; @@ -1006,19 +1034,20 @@ ntoa(undefined) -> undefined; ntoa({IpAddr, Port}) -> iolist_to_binary([inet:ntoa(IpAddr), ":", integer_to_list(Port)]); ntoa(IpAddr) -> iolist_to_binary(inet:ntoa(IpAddr)). -event_name(<<"$events/client_connected", _/binary>>) -> 'client.connected'; -event_name(<<"$events/client_disconnected", _/binary>>) -> 'client.disconnected'; -event_name(<<"$events/client_connack", _/binary>>) -> 'client.connack'; -event_name(<<"$events/client_check_authz_complete", _/binary>>) -> 'client.check_authz_complete'; -event_name(<<"$events/session_subscribed", _/binary>>) -> 'session.subscribed'; -event_name(<<"$events/session_unsubscribed", _/binary>>) -> 'session.unsubscribed'; -event_name(<<"$events/message_delivered", _/binary>>) -> 'message.delivered'; -event_name(<<"$events/message_acked", _/binary>>) -> 'message.acked'; -event_name(<<"$events/message_dropped", _/binary>>) -> 'message.dropped'; -event_name(<<"$events/delivery_dropped", _/binary>>) -> 'delivery.dropped'; -event_name(<<"$bridges/", _/binary>> = Topic) -> Topic; +event_name(<<"$bridges/", _/binary>> = Bridge) -> Bridge; +event_name(<<"$events/client_connected">>) -> 'client.connected'; +event_name(<<"$events/client_disconnected">>) -> 'client.disconnected'; +event_name(<<"$events/client_connack">>) -> 'client.connack'; +event_name(<<"$events/client_check_authz_complete">>) -> 'client.check_authz_complete'; +event_name(<<"$events/session_subscribed">>) -> 'session.subscribed'; +event_name(<<"$events/session_unsubscribed">>) -> 'session.unsubscribed'; +event_name(<<"$events/message_delivered">>) -> 'message.delivered'; +event_name(<<"$events/message_acked">>) -> 'message.acked'; +event_name(<<"$events/message_dropped">>) -> 'message.dropped'; +event_name(<<"$events/delivery_dropped">>) -> 'delivery.dropped'; event_name(_) -> 'message.publish'. +event_topic(<<"$bridges/", _/binary>> = Bridge) -> Bridge; event_topic('client.connected') -> <<"$events/client_connected">>; event_topic('client.disconnected') -> <<"$events/client_disconnected">>; event_topic('client.connack') -> <<"$events/client_connack">>; @@ -1029,8 +1058,7 @@ event_topic('message.delivered') -> <<"$events/message_delivered">>; event_topic('message.acked') -> <<"$events/message_acked">>; event_topic('message.dropped') -> <<"$events/message_dropped">>; event_topic('delivery.dropped') -> <<"$events/delivery_dropped">>; -event_topic('message.publish') -> <<"$events/message_publish">>; -event_topic(<<"$bridges/", _/binary>> = Topic) -> Topic. +event_topic('message.publish') -> <<"$events/message_publish">>. printable_maps(undefined) -> #{}; diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl index c9774b93d..4a1c9d6f5 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl @@ -8,19 +8,18 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_mod_hook_fun(_) -> - Funcs = emqx_rule_events:module_info(exports), - [ - ?assert(lists:keymember(emqx_rule_events:hook_fun(Event), 1, Funcs)) - || Event <- [ - 'client.connected', - 'client.disconnected', - 'session.subscribed', - 'session.unsubscribed', - 'message.acked', - 'message.dropped', - 'message.delivered' - ] - ]. + Events = emqx_rule_events:event_names(), + lists:foreach( + fun(E) -> + ?assert(is_function(emqx_rule_events:hook_fun(E))) + end, + Events + ), + ?assertEqual( + fun emqx_rule_events:on_bridge_message_received/2, + emqx_rule_events:hook_fun(<<"$bridges/foo">>) + ), + ?assertError({invalid_event, foo}, emqx_rule_events:hook_fun(foo)). t_printable_maps(_) -> Headers = #{ @@ -42,3 +41,24 @@ t_printable_maps(_) -> ?assertNot(maps:is_key(redispatch_to, Converted)), ?assertNot(maps:is_key(shared_dispatch_ack, Converted)), ok. + +t_event_name_topic_conversion(_) -> + Events = emqx_rule_events:event_names() -- ['message.publish'], + Topics = [atom_to_binary(A) || A <- emqx_rule_events:event_topics_enum()], + Zip = lists:zip(Events, Topics), + lists:foreach( + fun({Event, Topic}) -> + ?assertEqual(Event, emqx_rule_events:event_name(Topic)), + ?assertEqual(Topic, emqx_rule_events:event_topic(Event)) + end, + Zip + ). + +t_special_events_name_topic_conversion(_) -> + Bridge = <<"$bridges/foo:bar">>, + AdHoc = <<"foo/bar">>, + NonExisting = <<"$events/message_publish">>, + ?assertEqual(Bridge, emqx_rule_events:event_name(Bridge)), + ?assertEqual('message.publish', emqx_rule_events:event_name(AdHoc)), + ?assertEqual('message.publish', emqx_rule_events:event_name(NonExisting)), + ?assertEqual(NonExisting, emqx_rule_events:event_topic('message.publish')).