Merge branch 'master' into authn-config-check

.ci/docker-compose-file/redis/.gitignore

@@ -0,0 +1,3 @@
+r7000i.log
+r7001i.log
+r7002i.log

.github/actions/package-macos/action.yaml

@@ -0,0 +1,91 @@
+name: 'Create MacOS package'
+inputs:
+  profile: # emqx, emqx-enterprise
+    required: true
+    type: string
+  otp: # 24.2.1-1, 23.3.4.9-3
+    required: true
+    type: string
+  os:
+    required: false
+    type: string
+    default: macos-11
+  apple_id_password:
+    required: true
+    type: string
+  apple_developer_identity:
+    required: true
+    type: string
+  apple_developer_id_bundle:
+    required: true
+    type: string
+  apple_developer_id_bundle_password:
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: prepare
+      shell: bash
+      run: |
+        brew update
+        brew install curl zip unzip kerl coreutils openssl@1.1
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        echo "/usr/local/bin" >> $GITHUB_PATH
+    - uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.kerl/${{ inputs.otp }}
+        key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
+    - name: build erlang
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        KERL_BUILD_BACKEND: git
+        OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
+      run: |
+        kerl update releases
+        kerl build ${{ inputs.otp }}
+        kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
+    - name: build ${{ inputs.profile }}
+      env:
+        AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
+        APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
+      shell: bash
+      run: |
+        . $HOME/.kerl/${{ inputs.otp }}/activate
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        make ${{ inputs.profile }}-tgz
+    - name: test ${{ inputs.profile }}
+      shell: bash
+      run: |
+        pkg_name=$(find _packages/${{ inputs.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
+        mkdir emqx
+        unzip -d emqx $pkg_name > /dev/null
+        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        ready='no'
+        for i in {1..30}; do
+          if curl -fs 127.0.0.1:18083/status > /dev/null; then
+            ready='yes'
+            break
+          fi
+          sleep 1
+        done
+        if [ "$ready" != "yes" ]; then
+          echo "Timed out waiting for emqx to be ready"
+          cat emqx/log/erlang.log.1
+          exit 1
+        fi
+        ./emqx/bin/emqx_ctl status
+        ./emqx/bin/emqx stop
+        rm -rf emqx

.github/workflows/build_packages.yaml

@@ -150,72 +150,26 @@ jobs:
         name: source
         path: .
     - name: unzip source code
-      run: unzip -q source.zip
+      run: |
+        ln -s . source
+        unzip -q source.zip
+        rm source source.zip
     - name: prepare
       run: |
-        brew update
-        brew install curl zip unzip kerl coreutils
-        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
-        echo "/usr/local/bin" >> $GITHUB_PATH
         git config --global credential.helper store
-    - uses: actions/cache@v2
+    - uses: ./.github/actions/package-macos
-      id: cache
       with:
-        path: ~/.kerl/${{ matrix.otp }}
+        profile: ${{ matrix.profile }}
-        key: otp-install-${{ matrix.otp }}-${{ matrix.os }}
+        otp: ${{ matrix.otp }}
-    - name: build erlang
+        os: ${{ matrix.os }}
-      if: steps.cache.outputs.cache-hit != 'true'
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-      timeout-minutes: 60
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-      env:
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        KERL_BUILD_BACKEND: git
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.otp }}
-        kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
-
-    - name: build
-      working-directory: source
-      env:
-        AUTO_INSTALL_BUILD_DEPS: 1
-        APPLE_SIGN_BINARIES: 1
-        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      run: |
-        . $HOME/.kerl/${{ matrix.otp }}/activate
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        rm -rf _build/${{ matrix.profile }}/lib
-        make ${{ matrix.profile }}-tgz
-    - name: test
-      working-directory: source
-      run: |
-        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
-        mkdir -p emqx
-        tar -C emqx -zxf $pkg_name
-        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..18}; do
-          if curl -fs 127.0.0.1:18083/status > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
     - uses: actions/upload-artifact@v1
       with:
         name: ${{ matrix.profile }}-${{ matrix.otp }}
-        path: source/_packages/${{ matrix.profile }}/.
+        path: _packages/${{ matrix.profile }}/.
 
   linux:
     needs: prepare

.github/workflows/build_slim_packages.yaml

@@ -133,75 +133,26 @@ jobs:
         - emqx-enterprise
         otp:
         - 24.2.1-1
-        macos:
+        os:
         - macos-11
 
-    runs-on: ${{ matrix.macos }}
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v2
     - name: prepare
       run: |
-        brew update
-        brew install curl zip unzip kerl coreutils openssl@1.1
-        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
-        echo "/usr/local/bin" >> $GITHUB_PATH
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
         echo "BUILD_WITH_QUIC=1" >> $GITHUB_ENV
-    - uses: actions/cache@v2
+    - uses: ./.github/actions/package-macos
-      id: cache
       with:
-        path: ~/.kerl/${{ matrix.otp }}
+        profile: ${{ matrix.profile }}
-        key: otp-install-${{ matrix.otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
+        otp: ${{ matrix.otp }}
-    - name: build erlang
+        os: ${{ matrix.os }}
-      if: steps.cache.outputs.cache-hit != 'true'
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-      timeout-minutes: 60
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-      env:
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        KERL_BUILD_BACKEND: git
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.otp }}
-        kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
-    - name: build ${{ matrix.profile }}
-      env:
-        AUTO_INSTALL_BUILD_DEPS: 1
-        APPLE_SIGN_BINARIES: 1
-        APPLE_ID: developers@emqx.io
-        APPLE_TEAM_ID: 26N6HYJLZA
-        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      run: |
-        . $HOME/.kerl/${{ matrix.otp }}/activate
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        make ${{ matrix.profile }}-tgz
-    - name: test
-      run: |
-        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
-        mkdir emqx
-        unzip -d emqx $pkg_name > /dev/null
-        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..30}; do
-          if curl -fs 127.0.0.1:18083/status > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
     - uses: actions/upload-artifact@v2
       with:
         name: macos

.gitignore

@@ -68,4 +68,3 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi
 # rendered configurations
 *.conf.rendered
 lux_logs/
-.ci/docker-compose-file/redis/*.log

CHANGES-5.0.md

@@ -1,6 +1,18 @@
 # 5.0.9
+
+## Enhancements
+
+* Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973)
+* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060)
+* More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136)
+
 ## Bug fixes
 
+* Check ACLs for last will testament topic before publishing the message. [#8930](https://github.com/emqx/emqx/pull/8930)
+* Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002)
+* Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963)
+* Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986)
+* Redispatch shared subscription messages. [#9104](https://github.com/emqx/emqx/pull/9104)
 * Ensure authentication type is an array, not struct. [#8923](https://github.com/emqx/emqx/pull/8923)
 
 # 5.0.8
@@ -16,14 +28,21 @@
 
 * Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926)
 * Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911)
+* Fix that redis authn will deny the unknown users [#8934](https://github.com/emqx/emqx/pull/8934)
+* Fix ExProto UDP client keepalive checking error.
+  This causes the clients to not expire as long as a new UDP packet arrives [#8866](https://github.com/emqx/emqx/pull/8866)
+* Fix that MQTT Bridge message payload could be empty string. [#8949](https://github.com/emqx/emqx/pull/8949)
 
 ## Enhancements
 
 * Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905)
 * Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823)
+* Don't allow updating config items when they already exist in `local-override.conf`. [#8851](https://github.com/emqx/emqx/pull/8851)
 * Remove `node.etc_dir` from emqx.conf, because it is never used.
   Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892)
 * Added a new API `POST /listeners` for creating listener. [#8876](https://github.com/emqx/emqx/pull/8876)
+* Close ExProto client process immediately if it's keepalive timeouted. [#8866](https://github.com/emqx/emqx/pull/8866)
+* Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8866](https://github.com/emqx/emqx/pull/8866)
 
 # 5.0.7
 

Dockerfile.ubuntu20.04.runner

@@ -27,15 +27,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
 
 # emqx will occupy these port:
 # - 1883 port for MQTT
-# - 8081 for mgmt API
 # - 8083 for WebSocket/HTTP
 # - 8084 for WSS/HTTPS
 # - 8883 port for MQTT(SSL)
 # - 11883 port for internal MQTT/TCP
-# - 18083 for dashboard
+# - 18083 for dashboard and API
 # - 4370 default Erlang distrbution port
 # - 5369 for backplain gen_rpc
-EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369
+EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Makefile

@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.0.8
+export EMQX_DASHBOARD_VERSION ?= v1.0.9
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1

README-CN.md

@@ -32,12 +32,6 @@ EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个
 docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
 ```
 
-或直接试用 EMQX 企业版（已内置 10 个并发连接的永不过期 License）
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
-```
-
 接下来请参考 [入门指南](https://www.emqx.io/docs/zh/v5.0/getting-started/getting-started.html#启动-emqx) 开启您的 EMQX 之旅。
 
 #### 在 Kubernetes 上运行 EMQX 集群
@@ -112,6 +106,27 @@ make
 _build/emqx/rel/emqx/bin/emqx console
 ```
 
+### 在 Apple 芯片（M1,M2）上编译
+
+基于 Apple 芯片的 Homebrew 将[默认的 home 目录](https://github.com/Homebrew/brew/issues/9177)从 `/usr/local` 改成了 `/opt/homebrew`，这个改变导致了一些兼容性问题。
+
+具体到 EMQX 来说，主要影响的是 `unixodbc`，如果使用 Homebrew 安装的 `unixodbc` 包，那么在使用 [kerl](https://github.com/kerl/kerl) 编译 Erlang/OTP 的时候，kerl 会找不到 `unixodbc`。
+
+解决此问题的方法如下:
+
+```bash
+brew install unixodbc kerl
+sudo ln -s $(realpath $(brew --prefix unixodbc)) /usr/local/odbc
+export CC="/usr/bin/gcc -I$(brew --prefix unixodbc)/include"
+export LDFLAGS="-L$(brew --prefix unixodbc)/lib"
+kerl build 24.3
+mkdir ~/.kerl/installations
+kerl install 24.3 ~/.kerl/installations/24.3
+. ~/.kerl/installations/24.3/activate
+```
+
+然后再使用 `make` 继续编译就可以了。
+
 ## 开源许可
 
 详见 [LICENSE](./LICENSE)。

README-RU.md

@@ -28,13 +28,7 @@
 #### Установка EMQX с помощью Docker
 
 ```
-docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
+docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
-```
-
-Или запустите EMQX Enterprise со встроенной бессрочной лицензией на 10 соединений.
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
 ```
 
 Чтобы ознакомиться с функциональностью EMQX, пожалуйста, следуйте [руководству по началу работы](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx).

README.md

@@ -33,12 +33,6 @@ The simplest way to set up EMQX is to create a managed deployment with EMQX Clou
 docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
 ```
 
-Or install EMQX Enterprise with a built-in license for ten connections that never expire.
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
-```
-
 Next, please follow the [getting started guide](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx) to tour the EMQX features.
 
 #### Run EMQX cluster on kubernetes

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.7").
+-define(EMQX_RELEASE_CE, "5.0.8").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

apps/emqx/rebar.config

@@ -27,7 +27,7 @@
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

apps/emqx/src/emqx.app.src

@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.8"},
+    {vsn, "5.0.9"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_channel.erl

@@ -345,7 +345,8 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
                 fun check_connect/2,
                 fun enrich_client/2,
                 fun set_log_meta/2,
-                fun check_banned/2
+                fun check_banned/2,
+                fun count_flapping_event/2
             ],
             ConnPkt,
             Channel#channel{conn_state = connecting}
@@ -354,12 +355,14 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
         {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
             ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
             NChannel1 = NChannel#channel{
-                will_msg = emqx_packet:will_msg(NConnPkt),
                 alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
             },
             case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of
                 {ok, Properties, NChannel2} ->
-                    process_connect(Properties, NChannel2);
+                    %% only store will_msg after successful authn
+                    %% fix for: https://github.com/emqx/emqx/issues/8886
+                    NChannel3 = NChannel2#channel{will_msg = emqx_packet:will_msg(NConnPkt)},
+                    process_connect(Properties, NChannel3);
                 {continue, Properties, NChannel2} ->
                     handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2);
                 {error, ReasonCode} ->
@@ -995,8 +998,13 @@ maybe_nack(Delivers) ->
     lists:filter(fun not_nacked/1, Delivers).
 
 not_nacked({deliver, _Topic, Msg}) ->
-    not (emqx_shared_sub:is_ack_required(Msg) andalso
+    case emqx_shared_sub:is_ack_required(Msg) of
-        (ok == emqx_shared_sub:nack_no_connection(Msg))).
+        true ->
+            ok = emqx_shared_sub:nack_no_connection(Msg),
+            false;
+        false ->
+            true
+    end.
 
 maybe_mark_as_delivered(Session, Delivers) ->
     case emqx_session:info(is_persistent, Session) of
@@ -1165,10 +1173,11 @@ handle_call(
     Channel = #channel{
         conn_state = ConnState,
         will_msg = WillMsg,
+        clientinfo = ClientInfo,
         conninfo = #{proto_ver := ProtoVer}
     }
 ) ->
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+    (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
     Channel1 =
         case ConnState of
             connected -> ensure_disconnected(kicked, Channel);
@@ -1219,6 +1228,8 @@ handle_call(
     ChanInfo1 = info(NChannel),
     emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}),
     reply(ok, reset_timer(alive_timer, NChannel));
+handle_call(get_mqueue, Channel) ->
+    reply({ok, get_mqueue(Channel)}, Channel);
 handle_call(Req, Channel) ->
     ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     reply(ignored, Channel).
@@ -1250,14 +1261,11 @@ handle_info(
     {sock_closed, Reason},
     Channel =
         #channel{
-            conn_state = ConnState,
+            conn_state = ConnState
-            clientinfo = ClientInfo = #{zone := Zone}
         }
 ) when
     ConnState =:= connected orelse ConnState =:= reauthenticating
 ->
-    emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
-        emqx_flapping:detect(ClientInfo),
     Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
     case maybe_shutdown(Reason, Channel1) of
         {ok, Channel2} -> {ok, {event, disconnected}, Channel2};
@@ -1359,8 +1367,10 @@ handle_timeout(
     end;
 handle_timeout(_TRef, expire_session, Channel) ->
     shutdown(expired, Channel);
-handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) ->
+handle_timeout(
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+    _TRef, will_message, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}
+) ->
+    (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
     {ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})};
 handle_timeout(
     _TRef,
@@ -1434,12 +1444,14 @@ terminate({shutdown, kicked}, Channel) ->
     run_terminate_hook(kicked, Channel);
 terminate({shutdown, Reason}, Channel) when
     Reason =:= discarded;
-    Reason =:= takenover;
+    Reason =:= takenover
-    Reason =:= not_authorized
 ->
     run_terminate_hook(Reason, Channel);
-terminate(Reason, Channel = #channel{will_msg = WillMsg}) ->
+terminate(Reason, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+    %% since will_msg is set to undefined as soon as it is published,
+    %% if will_msg still exists when the session is terminated, it
+    %% must be published immediately.
+    WillMsg =/= undefined andalso publish_will_msg(ClientInfo, WillMsg),
     (Reason =:= expired) andalso persist_if_session(Channel),
     run_terminate_hook(Reason, Channel).
 
@@ -1622,6 +1634,14 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
         false -> ok
     end.
 
+%%--------------------------------------------------------------------
+%% Flapping
+
+count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
+    emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
+        emqx_flapping:detect(ClientInfo),
+    {ok, Channel}.
+
 %%--------------------------------------------------------------------
 %% Authenticate
 
@@ -2098,10 +2118,10 @@ ensure_disconnected(
 
 maybe_publish_will_msg(Channel = #channel{will_msg = undefined}) ->
     Channel;
-maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg}) ->
+maybe_publish_will_msg(Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
     case will_delay_interval(WillMsg) of
         0 ->
-            ok = publish_will_msg(WillMsg),
+            ok = publish_will_msg(ClientInfo, WillMsg),
             Channel#channel{will_msg = undefined};
         I ->
             ensure_timer(will_timer, timer:seconds(I), Channel)
@@ -2114,9 +2134,19 @@ will_delay_interval(WillMsg) ->
         0
     ).
 
-publish_will_msg(Msg) ->
+publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
-    _ = emqx_broker:publish(Msg),
+    case emqx_access_control:authorize(ClientInfo, publish, Topic) of
-    ok.
+        allow ->
+            _ = emqx_broker:publish(Msg),
+            ok;
+        deny ->
+            ?tp(
+                warning,
+                last_will_testament_publish_denied,
+                #{topic => Topic}
+            ),
+            ok
+    end.
 
 %%--------------------------------------------------------------------
 %% Disconnect Reason
@@ -2207,3 +2237,6 @@ get_mqtt_conf(Zone, Key, Default) ->
 set_field(Name, Value, Channel) ->
     Pos = emqx_misc:index_of(Name, record_info(fields, channel)),
     setelement(Pos + 1, Channel, Value).
+
+get_mqueue(#channel{session = Session}) ->
+    emqx_session:get_mqueue(Session).

apps/emqx/src/emqx_config.erl

@@ -476,7 +476,7 @@ read_override_conf(#{} = Opts) ->
 
 override_conf_file(Opts) when is_map(Opts) ->
     Key =
-        case maps:get(override_to, Opts, local) of
+        case maps:get(override_to, Opts, cluster) of
             local -> local_override_conf_file;
             cluster -> cluster_override_conf_file
         end,

apps/emqx/src/emqx_config_handler.erl

@@ -43,6 +43,7 @@
     terminate/2,
     code_change/3
 ]).
+-export([is_mutable/3]).
 
 -define(MOD, {mod}).
 -define(WKEY, '?').
@@ -229,15 +230,26 @@ process_update_request([_], _Handlers, {remove, _Opts}) ->
 process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
     BinKeyPath = bin_path(ConfKeyPath),
-    NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
+    case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of
-    OverrideConf = remove_from_override_config(BinKeyPath, Opts),
+        allow ->
-    {ok, NewRawConf, OverrideConf, Opts};
+            NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
+            OverrideConf = remove_from_override_config(BinKeyPath, Opts),
+            {ok, NewRawConf, OverrideConf, Opts};
+        {deny, Reason} ->
+            {error, {permission_denied, Reason}}
+    end;
 process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
     case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
         {ok, NewRawConf} ->
-            OverrideConf = update_override_config(NewRawConf, Opts),
+            BinKeyPath = bin_path(ConfKeyPath),
-            {ok, NewRawConf, OverrideConf, Opts};
+            case check_permissions(update, BinKeyPath, NewRawConf, Opts) of
+                allow ->
+                    OverrideConf = update_override_config(NewRawConf, Opts),
+                    {ok, NewRawConf, OverrideConf, Opts};
+                {deny, Reason} ->
+                    {error, {permission_denied, Reason}}
+            end;
         Error ->
             Error
     end.
@@ -272,12 +284,11 @@ check_and_save_configs(
     UpdateArgs,
     Opts
 ) ->
-    OldConf = emqx_config:get_root(ConfKeyPath),
     Schema = schema(SchemaModule, ConfKeyPath),
     {AppEnvs, NewConf} = emqx_config:check_config(Schema, NewRawConf),
+    OldConf = emqx_config:get_root(ConfKeyPath),
     case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
         {ok, Result0} ->
-            remove_from_local_if_cluster_change(ConfKeyPath, Opts),
             ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts),
             Result1 = return_change_result(ConfKeyPath, UpdateArgs),
             {ok, Result1#{post_config_update => Result0}};
@@ -430,16 +441,6 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf)
 merge_to_old_config(UpdateReq, _RawConf) ->
     {ok, UpdateReq}.
 
-%% local-override.conf priority is higher than cluster-override.conf
-%% If we want cluster to take effect, we must remove the local.
-remove_from_local_if_cluster_change(BinKeyPath, #{override_to := cluster} = Opts) ->
-    Opts1 = Opts#{override_to => local},
-    Local = remove_from_override_config(BinKeyPath, Opts1),
-    _ = emqx_config:save_to_override_conf(Local, Opts1),
-    ok;
-remove_from_local_if_cluster_change(_BinKeyPath, _Opts) ->
-    ok.
-
 remove_from_override_config(_BinKeyPath, #{persistent := false}) ->
     undefined;
 remove_from_override_config(BinKeyPath, Opts) ->
@@ -544,3 +545,98 @@ load_prev_handlers() ->
 
 save_handlers(Handlers) ->
     application:set_env(emqx, ?MODULE, Handlers).
+
+check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) ->
+    allow;
+check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) ->
+    case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of
+        {ok, NewRaw} ->
+            LocalOverride = emqx_config:read_override_conf(#{override_to => local}),
+            case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of
+                {ok, LocalRaw} ->
+                    case is_mutable(Action, NewRaw, LocalRaw) of
+                        ok ->
+                            allow;
+                        {error, Error} ->
+                            ?SLOG(error, #{
+                                msg => "prevent_remove_local_override_conf",
+                                config_key_path => ConfKeyPath,
+                                error => Error
+                            }),
+                            {deny, "Disable changed from local-override.conf"}
+                    end;
+                {not_found, _, _} ->
+                    allow
+            end;
+        {not_found, _, _} ->
+            allow
+    end.
+
+is_mutable(Action, NewRaw, LocalRaw) ->
+    try
+        KeyPath = [],
+        is_mutable(KeyPath, Action, NewRaw, LocalRaw)
+    catch
+        throw:Error -> Error
+    end.
+
+-define(REMOVE_FAILED, "remove_failed").
+-define(UPDATE_FAILED, "update_failed").
+
+is_mutable(KeyPath, Action, New = #{}, Local = #{}) ->
+    maps:foreach(
+        fun(Key, SubLocal) ->
+            case maps:find(Key, New) of
+                error -> ok;
+                {ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal)
+            end
+        end,
+        Local
+    );
+is_mutable(KeyPath, remove, Update, Origin) ->
+    throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}});
+is_mutable(_KeyPath, update, Val, Val) ->
+    ok;
+is_mutable(KeyPath, update, Update, Origin) ->
+    throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+is_mutable_update_test() ->
+    Action = update,
+    ?assertEqual(ok, is_mutable(Action, #{}, #{})),
+    ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})),
+    ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})),
+    ?assertEqual(
+        {error, {?UPDATE_FAILED, [a, b, c], 1, 2}},
+        is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
+    ),
+    ?assertEqual(
+        {error, {?UPDATE_FAILED, [a, b, d], 2, 3}},
+        is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
+    ),
+    ok.
+
+is_mutable_remove_test() ->
+    Action = remove,
+    ?assertEqual(ok, is_mutable(Action, #{}, #{})),
+    ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})),
+    ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})),
+    ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})),
+
+    ?assertEqual(
+        {error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
+        is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})
+    ),
+    ?assertEqual(
+        {error, {?REMOVE_FAILED, [a, b, c], 1, 2}},
+        is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
+    ),
+    ?assertEqual(
+        {error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
+        is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
+    ),
+    ok.
+
+-endif.

apps/emqx/src/emqx_listeners.erl

@@ -87,12 +87,18 @@ format_list(Listener) ->
     ].
 
 do_list_raw() ->
-    Key = <<"listeners">>,
+    %% GET /listeners from other nodes returns [] when init config is not loaded.
-    Raw = emqx_config:get_raw([Key], #{}),
+    case emqx_app:get_init_config_load_done() of
-    SchemaMod = emqx_config:get_schema_mod(Key),
+        true ->
-    #{Key := RawWithDefault} = emqx_config:fill_defaults(SchemaMod, #{Key => Raw}, #{}),
+            Key = <<"listeners">>,
-    Listeners = maps:to_list(RawWithDefault),
+            Raw = emqx_config:get_raw([Key], #{}),
-    lists:flatmap(fun format_raw_listeners/1, Listeners).
+            SchemaMod = emqx_config:get_schema_mod(Key),
+            #{Key := RawWithDefault} = emqx_config:fill_defaults(SchemaMod, #{Key => Raw}, #{}),
+            Listeners = maps:to_list(RawWithDefault),
+            lists:flatmap(fun format_raw_listeners/1, Listeners);
+        false ->
+            []
+    end.
 
 format_raw_listeners({Type0, Conf}) ->
     Type = binary_to_atom(Type0),
@@ -515,12 +521,16 @@ merge_default(Options) ->
     integer() | {tuple(), integer()} | string() | binary()
 ) -> io_lib:chars().
 format_bind(Port) when is_integer(Port) ->
+    %% **Note**:
+    %% 'For TCP, UDP and IP networks, if the host is empty or a literal
+    %% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for
+    %% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is
+    %% assumed.'
+    %%
+    %% Quoted from: https://pkg.go.dev/net
+    %% Decided to use this format to display the bind for all interfaces and
+    %% IPv4/IPv6 support
     io_lib:format(":~w", [Port]);
-%% Print only the port number when bound on all interfaces
-format_bind({{0, 0, 0, 0}, Port}) ->
-    format_bind(Port);
-format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) ->
-    format_bind(Port);
 format_bind({Addr, Port}) when is_list(Addr) ->
     io_lib:format("~ts:~w", [Addr, Port]);
 format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 ->
@@ -532,6 +542,8 @@ format_bind(Str) when is_list(Str) ->
     case emqx_schema:to_ip_port(Str) of
         {ok, {Ip, Port}} ->
             format_bind({Ip, Port});
+        {ok, Port} ->
+            format_bind(Port);
         {error, _} ->
             format_bind(list_to_integer(Str))
     end;

apps/emqx/src/emqx_mqueue.erl

@@ -66,7 +66,8 @@
     in/2,
     out/1,
     stats/1,
-    dropped/1
+    dropped/1,
+    to_list/1
 ]).
 
 -define(NO_PRIORITY_TABLE, disabled).
@@ -109,7 +110,7 @@
     dropped = 0 :: count(),
     p_table = ?NO_PRIORITY_TABLE :: p_table(),
     default_p = ?LOWEST_PRIORITY :: priority(),
-    q = ?PQUEUE:new() :: pq(),
+    q = emqx_pqueue:new() :: pq(),
     shift_opts :: #shift_opts{},
     last_prio :: non_neg_integer() | undefined,
     p_credit :: non_neg_integer() | undefined
@@ -118,7 +119,7 @@
 -type mqueue() :: #mqueue{}.
 
 -spec init(options()) -> mqueue().
-init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
+init(Opts = #{max_len := MaxLen0, store_qos0 := Qos0}) ->
     MaxLen =
         case (is_integer(MaxLen0) andalso MaxLen0 > ?MAX_LEN_INFINITY) of
             true -> MaxLen0;
@@ -126,7 +127,7 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
         end,
     #mqueue{
         max_len = MaxLen,
-        store_qos0 = QoS_0,
+        store_qos0 = Qos0,
         p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE),
         default_p = get_priority_opt(Opts),
         shift_opts = get_shift_opt(Opts)
@@ -152,6 +153,19 @@ len(#mqueue{len = Len}) -> Len.
 
 max_len(#mqueue{max_len = MaxLen}) -> MaxLen.
 
+%% @doc Return all queued items in a list.
+-spec to_list(mqueue()) -> list().
+to_list(MQ) ->
+    to_list(MQ, []).
+
+to_list(MQ, Acc) ->
+    case out(MQ) of
+        {empty, _MQ} ->
+            lists:reverse(Acc);
+        {{value, Msg}, Q1} ->
+            to_list(Q1, [Msg | Acc])
+    end.
+
 %% @doc Return number of dropped messages.
 -spec dropped(mqueue()) -> count().
 dropped(#mqueue{dropped = Dropped}) -> Dropped.

apps/emqx/src/emqx_schema.erl

@@ -39,7 +39,8 @@
 -type comma_separated_binary() :: [binary()].
 -type comma_separated_atoms() :: [atom()].
 -type bar_separated_list() :: list().
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
+-type host_port() :: tuple().
 -type cipher() :: map().
 
 -typerefl_from_string({duration/0, emqx_schema, to_duration}).
@@ -52,6 +53,7 @@
 -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}).
 -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}).
 -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
+-typerefl_from_string({host_port/0, emqx_schema, to_host_port}).
 -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
 -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
 
@@ -78,6 +80,7 @@
     to_comma_separated_binary/1,
     to_bar_separated_list/1,
     to_ip_port/1,
+    to_host_port/1,
     to_erl_cipher_suite/1,
     to_comma_separated_atoms/1
 ]).
@@ -96,6 +99,7 @@
     comma_separated_binary/0,
     bar_separated_list/0,
     ip_port/0,
+    host_port/0,
     cipher/0,
     comma_separated_atoms/0
 ]).
@@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) ->
 %%  - :1883
 %%  - :::1883
 to_ip_port(Str) ->
-    case split_ip_port(Str) of
+    to_host_port(Str, ip_addr).
-        {"", Port} ->
+
-            {ok, {{0, 0, 0, 0}, list_to_integer(Port)}};
+%% @doc support the following format:
-        {Ip, Port} ->
+%%  - 127.0.0.1:1883
+%%  - ::1:1883
+%%  - [::1]:1883
+%%  - :1883
+%%  - :::1883
+%%  - example.com:80
+to_host_port(Str) ->
+    to_host_port(Str, hostname).
+
+%%  - example.com:80
+to_host_port(Str, IpOrHost) ->
+    case split_host_port(Str) of
+        {"", Port} when IpOrHost =:= ip_addr ->
+            %% this is a local address
+            {ok, list_to_integer(Port)};
+        {"", _Port} ->
+            %% must specify host part when it's a remote endpoint
+            {error, bad_host_port};
+        {MaybeIp, Port} ->
             PortVal = list_to_integer(Port),
-            case inet:parse_address(Ip) of
+            case inet:parse_address(MaybeIp) of
-                {ok, R} ->
+                {ok, IpTuple} ->
-                    {ok, {R, PortVal}};
+                    {ok, {IpTuple, PortVal}};
-                _ ->
+                _ when IpOrHost =:= hostname ->
                     %% check is a rfc1035's hostname
-                    case inet_parse:domain(Ip) of
+                    case inet_parse:domain(MaybeIp) of
                         true ->
-                            {ok, {Ip, PortVal}};
+                            {ok, {MaybeIp, PortVal}};
                         _ ->
-                            {error, Str}
+                            {error, bad_hostname}
-                    end
+                    end;
+                _ ->
+                    {error, bad_ip_port}
             end;
         _ ->
-            {error, Str}
+            {error, bad_ip_port}
     end.
 
-split_ip_port(Str0) ->
+split_host_port(Str0) ->
     Str = re:replace(Str0, " ", "", [{return, list}, global]),
     case lists:split(string:rchr(Str, $:), Str) of
-        %% no port
+        %% no colon
         {[], Str} ->
-            error;
+            try
+                %% if it's just a port number, then return as-is
+                _ = list_to_integer(Str),
+                {"", Str}
+            catch
+                _:_ ->
+                    error
+            end;
         {IpPlusColon, PortString} ->
             IpStr0 = lists:droplast(IpPlusColon),
             case IpStr0 of

apps/emqx/src/emqx_session.erl

@@ -60,7 +60,8 @@
     info/2,
     is_session/1,
     stats/1,
-    obtain_next_pkt_id/1
+    obtain_next_pkt_id/1,
+    get_mqueue/1
 ]).
 
 -export([
@@ -801,7 +802,8 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) ->
 -spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok.
 terminate(ClientInfo, Reason, Session) ->
     run_terminate_hooks(ClientInfo, Reason, Session),
-    redispatch_shared_messages(Session),
+    Reason =/= takenover andalso
+        redispatch_shared_messages(Session),
     ok.
 
 run_terminate_hooks(ClientInfo, discarded, Session) ->
@@ -811,29 +813,20 @@ run_terminate_hooks(ClientInfo, takenover, Session) ->
 run_terminate_hooks(ClientInfo, Reason, Session) ->
     run_hook('session.terminated', [ClientInfo, Reason, info(Session)]).
 
-redispatch_shared_messages(#session{inflight = Inflight}) ->
+redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) ->
-    InflightList = emqx_inflight:to_list(Inflight),
+    AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight),
-    lists:foreach(
+    F = fun
-        fun
+        ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) ->
-            %% Only QoS1 messages get redispatched, because QoS2 messages
+            %% For QoS 2, here is what the spec says:
-            %% must be sent to the same client, once they're in flight
+            %% If the Client's Session terminates before the Client reconnects,
-            ({_, #inflight_data{message = #message{qos = ?QOS_2} = Msg}}) ->
+            %% the Server MUST NOT send the Application Message to any other
-                ?SLOG(warning, #{msg => qos2_lost_no_redispatch}, #{message => Msg});
+            %% subscribed Client [MQTT-4.8.2-5].
-            ({_, #inflight_data{message = #message{topic = Topic, qos = ?QOS_1} = Msg}}) ->
+            {true, Msg};
-                case emqx_shared_sub:get_group(Msg) of
+        ({_PacketId, #inflight_data{}}) ->
-                    {ok, Group} ->
+            false
-                        %% Note that dispatch is called with self() in failed subs
+    end,
-                        %% This is done to avoid dispatching back to caller
+    InflightList = lists:filtermap(F, AllInflights),
-                        Delivery = #delivery{sender = self(), message = Msg},
+    emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)).
-                        emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]);
-                    _ ->
-                        false
-                end;
-            (_) ->
-                ok
-        end,
-        InflightList
-    ).
 
 -compile({inline, [run_hook/2]}).
 run_hook(Name, Args) ->
@@ -892,7 +885,7 @@ on_delivery_completed(
     ).
 
 mark_begin_deliver(Msg) ->
-    emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg).
+    emqx_message:set_header(deliver_begin_at, erlang:system_time(millisecond), Msg).
 
 %%--------------------------------------------------------------------
 %% Helper functions
@@ -925,3 +918,6 @@ age(Now, Ts) -> Now - Ts.
 set_field(Name, Value, Session) ->
     Pos = emqx_misc:index_of(Name, record_info(fields, session)),
     setelement(Pos + 1, Session, Value).
+
+get_mqueue(#session{mqueue = Q}) ->
+    emqx_mqueue:to_list(Q).

apps/emqx/src/emqx_shared_sub.erl

@@ -39,15 +39,15 @@
 -export([
     dispatch/3,
     dispatch/4,
-    do_dispatch_with_ack/4
+    do_dispatch_with_ack/4,
+    redispatch/1
 ]).
 
 -export([
     maybe_ack/1,
     maybe_nack_dropped/1,
     nack_no_connection/1,
-    is_ack_required/1,
+    is_ack_required/1
-    get_group/1
 ]).
 
 %% for testing
@@ -96,6 +96,9 @@
 -define(ACK, shared_sub_ack).
 -define(NACK(Reason), {shared_sub_nack, Reason}).
 -define(NO_ACK, no_ack).
+-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
+
+-type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
 
 -record(state, {pmon}).
 
@@ -144,7 +147,8 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
         false ->
             {error, no_subscribers};
         {Type, SubPid} ->
-            case do_dispatch(SubPid, Group, Topic, Msg, Type) of
+            Msg1 = with_redispatch_to(Msg, Group, Topic),
+            case do_dispatch(SubPid, Group, Topic, Msg1, Type) of
                 ok ->
                     {ok, 1};
                 {error, _Reason} ->
@@ -223,16 +227,53 @@ without_group_ack(Msg) ->
 get_group_ack(Msg) ->
     emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
 
+with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
+    Msg;
+with_redispatch_to(Msg, Group, Topic) ->
+    emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
+
+%% @hidden Redispatch is neede only for the messages with redispatch_to header added.
+is_redispatch_needed(#message{} = Msg) ->
+    case get_redispatch_to(Msg) of
+        ?REDISPATCH_TO(_, _) ->
+            true;
+        _ ->
+            false
+    end.
+
+%% @doc Redispatch shared deliveries to other members in the group.
+redispatch(Messages0) ->
+    Messages = lists:filter(fun is_redispatch_needed/1, Messages0),
+    case length(Messages) of
+        L when L > 0 ->
+            ?SLOG(info, #{
+                msg => "redispatching_shared_subscription_message",
+                count => L
+            }),
+            lists:foreach(fun redispatch_shared_message/1, Messages);
+        _ ->
+            ok
+    end.
+
+redispatch_shared_message(#message{} = Msg) ->
+    %% As long as it's still a #message{} record in inflight,
+    %% we should try to re-dispatch
+    ?REDISPATCH_TO(Group, Topic) = get_redispatch_to(Msg),
+    %% Note that dispatch is called with self() in failed subs
+    %% This is done to avoid dispatching back to caller
+    Delivery = #delivery{sender = self(), message = Msg},
+    dispatch(Group, Topic, Delivery, [self()]).
+
+%% @hidden Return the `redispatch_to` group-topic in the message header.
+%% `false` is returned if the message is not a shared dispatch.
+%% or when it's a QoS 0 message.
+-spec get_redispatch_to(emqx_types:message()) -> redispatch_to() | false.
+get_redispatch_to(Msg) ->
+    emqx_message:get_header(redispatch_to, Msg, false).
+
 -spec is_ack_required(emqx_types:message()) -> boolean().
 is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg).
 
--spec get_group(emqx_types:message()) -> {ok, any()} | error.
-get_group(Msg) ->
-    case get_group_ack(Msg) of
-        ?NO_ACK -> error;
-        {Group, _Sender, _Ref} -> {ok, Group}
-    end.
-
 %% @doc Negative ack dropped message due to inflight window or message queue being full.
 -spec maybe_nack_dropped(emqx_types:message()) -> boolean().
 maybe_nack_dropped(Msg) ->

apps/emqx/test/emqx_channel_SUITE.erl

@@ -207,14 +207,6 @@ init_per_suite(Config) ->
     ok = meck:new(emqx_cm, [passthrough, no_history, no_link]),
     ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end),
     ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end),
-    %% Access Control Meck
-    ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
-    ok = meck:expect(
-        emqx_access_control,
-        authenticate,
-        fun(_) -> {ok, #{is_superuser => false}} end
-    ),
-    ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
     %% Broker Meck
     ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
     %% Hooks Meck
@@ -234,7 +226,6 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     meck:unload([
-        emqx_access_control,
         emqx_metrics,
         emqx_session,
         emqx_broker,
@@ -244,11 +235,21 @@ end_per_suite(_Config) ->
     ]).
 
 init_per_testcase(_TestCase, Config) ->
+    %% Access Control Meck
+    ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) -> {ok, #{is_superuser => false}} end
+    ),
+    ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
+    %% Set confs
     OldConf = set_test_listener_confs(),
     emqx_common_test_helpers:start_apps([]),
     [{config, OldConf} | Config].
 
 end_per_testcase(_TestCase, Config) ->
+    meck:unload([emqx_access_control]),
     emqx_config:put(?config(config, Config)),
     emqx_common_test_helpers:stop_apps([]),
     Config.
@@ -1115,6 +1116,32 @@ t_ws_cookie_init(_) ->
     ),
     ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)).
 
+%%--------------------------------------------------------------------
+%% Test cases for other mechnisms
+%%--------------------------------------------------------------------
+
+t_flapping_detect(_) ->
+    emqx_config:put_zone_conf(default, [flapping_detect, enable], true),
+    Parent = self(),
+    ok = meck:expect(
+        emqx_cm,
+        open_session,
+        fun(true, _ClientInfo, _ConnInfo) ->
+            {ok, #{session => session(), present => false}}
+        end
+    ),
+    ok = meck:expect(emqx_access_control, authenticate, fun(_) -> {error, not_authorized} end),
+    ok = meck:expect(emqx_flapping, detect, fun(_) -> Parent ! flapping_detect end),
+    IdleChannel = channel(#{conn_state => idle}),
+    {shutdown, not_authorized, _ConnAck, _Channel} =
+        emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel),
+    receive
+        flapping_detect -> ok
+    after 2000 ->
+        ?assert(false, "Flapping detect should be exected in connecting progress")
+    end,
+    meck:unload([emqx_flapping]).
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------

apps/emqx/test/emqx_common_test_helpers.erl

@@ -632,9 +632,9 @@ setup_node(Node, Opts) when is_map(Opts) ->
     %% Here we start the apps
     EnvHandlerForRpc =
         fun(App) ->
-            %% We load configuration, and than set the special enviroment variable
+            %% We load configuration, and than set the special environment variable
             %% which says that emqx shouldn't load configuration at startup
-            %% Otherwise, configuration get's loaded and all preset env in envhandler is lost
+            %% Otherwise, configuration gets loaded and all preset env in EnvHandler is lost
             LoadSchema andalso
                 begin
                     emqx_config:init_load(SchemaMod),

apps/emqx/test/emqx_config_handler_SUITE.erl

@@ -21,6 +21,8 @@
 
 -define(MOD, {mod}).
 -define(WKEY, '?').
+-define(LOCAL_CONF, "/tmp/local-override.conf").
+-define(CLUSTER_CONF, "/tmp/cluster-override.conf").
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -36,6 +38,8 @@ end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
 init_per_testcase(_Case, Config) ->
+    _ = file:delete(?LOCAL_CONF),
+    _ = file:delete(?CLUSTER_CONF),
     Config.
 
 end_per_testcase(_Case, _Config) ->
@@ -196,6 +200,62 @@ t_sub_key_update_remove(_Config) ->
     ok = emqx_config_handler:remove_handler(KeyPath2),
     ok.
 
+t_local_override_update_remove(_Config) ->
+    application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF),
+    application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF),
+    KeyPath = [sysmon, os, cpu_high_watermark],
+    ok = emqx_config_handler:add_handler(KeyPath, ?MODULE),
+    LocalOpts = #{override_to => local},
+    {ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts),
+    ?assertMatch(
+        #{
+            config := 0.7,
+            post_config_update := #{},
+            raw_config := <<"70%">>
+        },
+        Res
+    ),
+    ClusterOpts = #{override_to => cluster},
+    ?assertMatch(
+        {error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts)
+    ),
+    ?assertMatch(0.7, emqx:get_config(KeyPath)),
+
+    KeyPath2 = [sysmon, os, cpu_low_watermark],
+    ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE),
+    ?assertMatch(
+        {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts)
+    ),
+
+    %% remove
+    ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)),
+    ?assertEqual(
+        {ok, #{post_config_update => #{}}},
+        emqx:remove_config(KeyPath, #{override_to => local})
+    ),
+    ?assertEqual(
+        {ok, #{post_config_update => #{}}},
+        emqx:remove_config(KeyPath)
+    ),
+    ?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)),
+    OSKey = maps:keys(emqx:get_raw_config([sysmon, os])),
+    ?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)),
+    ?assert(length(OSKey) > 0),
+
+    ?assertEqual(
+        {ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}},
+        emqx:reset_config(KeyPath, ClusterOpts)
+    ),
+    OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])),
+    ?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)),
+    ?assert(length(OSKey1) > 1),
+
+    ok = emqx_config_handler:remove_handler(KeyPath),
+    ok = emqx_config_handler:remove_handler(KeyPath2),
+    application:unset_env(emqx, local_override_conf_file),
+    application:unset_env(emqx, cluster_override_conf_file),
+    ok.
+
 t_check_failed(_Config) ->
     KeyPath = [sysmon, os, cpu_check_interval],
     Opts = #{rawconf_with_defaults => true},
@@ -219,7 +279,7 @@ t_stop(_Config) ->
     ok.
 
 t_callback_crash(_Config) ->
-    CrashPath = [sysmon, os, cpu_high_watermark],
+    CrashPath = [sysmon, os, procmem_high_watermark],
     Opts = #{rawconf_with_defaults => true},
     ok = emqx_config_handler:add_handler(CrashPath, ?MODULE),
     Old = emqx:get_raw_config(CrashPath),
@@ -334,6 +394,8 @@ pre_config_update([sysmon, os, cpu_check_interval], UpdateReq, _RawConf) ->
     {ok, UpdateReq};
 pre_config_update([sysmon, os, cpu_low_watermark], UpdateReq, _RawConf) ->
     {ok, UpdateReq};
+pre_config_update([sysmon, os, cpu_high_watermark], UpdateReq, _RawConf) ->
+    {ok, UpdateReq};
 pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) ->
     {ok, UpdateReq};
 pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) ->
@@ -347,6 +409,8 @@ post_config_update([sysmon, os, cpu_check_interval], _UpdateReq, _NewConf, _OldC
     {ok, ok};
 post_config_update([sysmon, os, cpu_low_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
     ok;
+post_config_update([sysmon, os, cpu_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
+    ok;
 post_config_update([sysmon, os, sysmem_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
     {error, post_config_update_error}.
 

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -148,6 +148,32 @@ t_wss_conn(_) ->
     {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000),
     ok = ssl:close(Socket).
 
+t_format_bind(_) ->
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind(1883))
+    ),
+    ?assertEqual(
+        "0.0.0.0:1883",
+        lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883}))
+    ),
+    ?assertEqual(
+        "[::]:1883",
+        lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883}))
+    ),
+    ?assertEqual(
+        "127.0.0.1:1883",
+        lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883}))
+    ),
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind("1883"))
+    ),
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind(":1883"))
+    ).
+
 render_config_file() ->
     Path = local_path(["etc", "emqx.conf"]),
     {ok, Temp} = file:read_file(Path),

apps/emqx/test/emqx_schema_tests.erl

@@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() ->
         Checked
     ),
     ok.
+
+to_ip_port_test_() ->
+    Ip = fun emqx_schema:to_ip_port/1,
+    Host = fun(Str) ->
+        case Ip(Str) of
+            {ok, {_, _} = Res} ->
+                %% assert
+                {ok, Res} = emqx_schema:to_host_port(Str);
+            _ ->
+                emqx_schema:to_host_port(Str)
+        end
+    end,
+    [
+        ?_assertEqual({ok, 80}, Ip("80")),
+        ?_assertEqual({error, bad_host_port}, Host("80")),
+        ?_assertEqual({ok, 80}, Ip(":80")),
+        ?_assertEqual({error, bad_host_port}, Host(":80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")),
+        ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")),
+        ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")),
+        ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("$:1900")),
+        ?_assertEqual({error, bad_hostname}, Host("$:1900")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip(":::1883"))
+    ].

apps/emqx/test/emqx_shared_sub_SUITE.erl

@@ -25,10 +25,20 @@
 
 -define(SUITE, ?MODULE).
 
--define(wait(For, Timeout),
+-define(WAIT(TIMEOUT, PATTERN, Res),
-    emqx_common_test_helpers:wait_for(
+    (fun() ->
-        ?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout
+        receive
-    )
+            PATTERN ->
+                Res;
+            Other ->
+                ct:fail(#{
+                    expected => ??PATTERN,
+                    got => Other
+                })
+        after TIMEOUT ->
+            ct:fail({timeout, ??PATTERN})
+        end
+    end)()
 ).
 
 -define(ack, shared_sub_ack).
@@ -45,10 +55,26 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
-t_is_ack_required(_) ->
+init_per_testcase(Case, Config) ->
+    try
+        ?MODULE:Case({'init', Config})
+    catch
+        error:function_clause ->
+            Config
+    end.
+
+end_per_testcase(Case, Config) ->
+    try
+        ?MODULE:Case({'end', Config})
+    catch
+        error:function_clause ->
+            ok
+    end.
+
+t_is_ack_required(Config) when is_list(Config) ->
     ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
 
-t_maybe_nack_dropped(_) ->
+t_maybe_nack_dropped(Config) when is_list(Config) ->
     ?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
     Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
     ?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)),
@@ -60,7 +86,7 @@ t_maybe_nack_dropped(_) ->
         end
     ).
 
-t_nack_no_connection(_) ->
+t_nack_no_connection(Config) when is_list(Config) ->
     Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
     ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)),
     ?assertEqual(
@@ -71,7 +97,7 @@ t_nack_no_connection(_) ->
         end
     ).
 
-t_maybe_ack(_) ->
+t_maybe_ack(Config) when is_list(Config) ->
     ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})),
     Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
     ?assertEqual(
@@ -86,10 +112,7 @@ t_maybe_ack(_) ->
         end
     ).
 
-% t_subscribers(_) ->
+t_random_basic(Config) when is_list(Config) ->
-%     error('TODO').
-
-t_random_basic(_) ->
     ok = ensure_config(random),
     ClientId = <<"ClientId">>,
     Topic = <<"foo">>,
@@ -121,7 +144,7 @@ t_random_basic(_) ->
 %% After the connection for the 2nd session is also closed,
 %% i.e. when all clients are offline, the following message(s)
 %% should be delivered randomly.
-t_no_connection_nack(_) ->
+t_no_connection_nack(Config) when is_list(Config) ->
     ok = ensure_config(sticky),
     Publisher = <<"publisher">>,
     Subscriber1 = <<"Subscriber1">>,
@@ -153,54 +176,22 @@ t_no_connection_nack(_) ->
     %% This is the connection which was picked by broker to dispatch (sticky) for 1st message
 
     ?assertMatch([#{packet_id := 1}], recv_msgs(1)),
-    %% Now kill the connection, expect all following messages to be delivered to the other
-    %% subscriber.
-    %emqx_mock_client:stop(ConnPid),
-    %% sleep then make synced calls to session processes to ensure that
-    %% the connection pid's 'EXIT' message is propagated to the session process
-    %% also to be sure sessions are still alive
-    %   timer:sleep(2),
-    %   _ = emqx_session:info(SPid1),
-    %   _ = emqx_session:info(SPid2),
-    %   %% Now we know what is the other still alive connection
-    %   [TheOtherConnPid] = [SubConnPid1, SubConnPid2] -- [ConnPid],
-    %   %% Send some more messages
-    %   PacketIdList = lists:seq(2, 10),
-    %   lists:foreach(fun(Id) ->
-    %                         SendF(Id),
-    %                         ?wait(Received(Id, TheOtherConnPid), 1000)
-    %                 end, PacketIdList),
-    %   %% Now close the 2nd (last connection)
-    %   emqx_mock_client:stop(TheOtherConnPid),
-    %   timer:sleep(2),
-    %   %% both sessions should have conn_pid = undefined
-    %   ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid1))),
-    %   ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid2))),
-    %   %% send more messages, but all should be queued in session state
-    %   lists:foreach(fun(Id) -> SendF(Id) end, PacketIdList),
-    %   {_, L1} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid1)),
-    %   {_, L2} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid2)),
-    %   ?assertEqual(length(PacketIdList), L1 + L2),
-    %   %% clean up
-    %   emqx_mock_client:close_session(PubConnPid),
-    %   emqx_sm:close_session(SPid1),
-    %   emqx_sm:close_session(SPid2),
     ok.
 
-t_random(_) ->
+t_random(Config) when is_list(Config) ->
     ok = ensure_config(random, true),
     test_two_messages(random).
 
-t_round_robin(_) ->
+t_round_robin(Config) when is_list(Config) ->
     ok = ensure_config(round_robin, true),
     test_two_messages(round_robin).
 
-t_round_robin_per_group(_) ->
+t_round_robin_per_group(Config) when is_list(Config) ->
     ok = ensure_config(round_robin_per_group, true),
     test_two_messages(round_robin_per_group).
 
 %% this would fail if executed with the standard round_robin strategy
-t_round_robin_per_group_even_distribution_one_group(_) ->
+t_round_robin_per_group_even_distribution_one_group(Config) when is_list(Config) ->
     ok = ensure_config(round_robin_per_group, true),
     Topic = <<"foo/bar">>,
     Group = <<"group1">>,
@@ -264,7 +255,7 @@ t_round_robin_per_group_even_distribution_one_group(_) ->
     ),
     ok.
 
-t_round_robin_per_group_even_distribution_two_groups(_) ->
+t_round_robin_per_group_even_distribution_two_groups(Config) when is_list(Config) ->
     ok = ensure_config(round_robin_per_group, true),
     Topic = <<"foo/bar">>,
     {ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]),
@@ -350,19 +341,19 @@ t_round_robin_per_group_even_distribution_two_groups(_) ->
     ),
     ok.
 
-t_sticky(_) ->
+t_sticky(Config) when is_list(Config) ->
     ok = ensure_config(sticky, true),
     test_two_messages(sticky).
 
-t_hash(_) ->
+t_hash(Config) when is_list(Config) ->
     ok = ensure_config(hash, false),
     test_two_messages(hash).
 
-t_hash_clinetid(_) ->
+t_hash_clinetid(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
-t_hash_topic(_) ->
+t_hash_topic(Config) when is_list(Config) ->
     ok = ensure_config(hash_topic, false),
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
@@ -407,7 +398,7 @@ t_hash_topic(_) ->
     ok.
 
 %% if the original subscriber dies, change to another one alive
-t_not_so_sticky(_) ->
+t_not_so_sticky(Config) when is_list(Config) ->
     ok = ensure_config(sticky),
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
@@ -481,7 +472,7 @@ last_message(ExpectedPayload, Pids, Timeout) ->
         <<"not yet?">>
     end.
 
-t_dispatch(_) ->
+t_dispatch(Config) when is_list(Config) ->
     ok = ensure_config(random),
     Topic = <<"foo">>,
     ?assertEqual(
@@ -494,13 +485,13 @@ t_dispatch(_) ->
         emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
     ).
 
-t_uncovered_func(_) ->
+t_uncovered_func(Config) when is_list(Config) ->
     ignored = gen_server:call(emqx_shared_sub, ignored),
     ok = gen_server:cast(emqx_shared_sub, ignored),
     ignored = emqx_shared_sub ! ignored,
     {mnesia_table_event, []} = emqx_shared_sub ! {mnesia_table_event, []}.
 
-t_per_group_config(_) ->
+t_per_group_config(Config) when is_list(Config) ->
     ok = ensure_group_config(#{
         <<"local_group">> => local,
         <<"round_robin_group">> => round_robin,
@@ -521,7 +512,7 @@ t_per_group_config(_) ->
     test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>),
     test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>).
 
-t_local(_) ->
+t_local(Config) when is_list(Config) ->
     GroupConfig = #{
         <<"local_group">> => local,
         <<"round_robin_group">> => round_robin,
@@ -567,7 +558,7 @@ t_local(_) ->
     ?assertNotEqual(UsedSubPid1, UsedSubPid2),
     ok.
 
-t_remote(_) ->
+t_remote(Config) when is_list(Config) ->
     %% This testcase verifies dispatching of shared messages to the remote nodes via backplane API.
     %%
     %% In this testcase we start two EMQX nodes: local and remote.
@@ -594,7 +585,7 @@ t_remote(_) ->
 
     try
         {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal),
-        {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote),
+        {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote),
 
         emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}),
 
@@ -620,7 +611,7 @@ t_remote(_) ->
         stop_slave(Node)
     end.
 
-t_local_fallback(_) ->
+t_local_fallback(Config) when is_list(Config) ->
     ok = ensure_group_config(#{
         <<"local_group">> => local,
         <<"round_robin_group">> => round_robin,
@@ -653,9 +644,14 @@ t_local_fallback(_) ->
 
 %% This one tests that broker tries to select another shared subscriber
 %% If the first one doesn't return an ACK
-t_redispatch(_) ->
+t_redispatch_qos1_with_ack(Config) when is_list(Config) ->
-    ok = ensure_config(sticky, true),
+    test_redispatch_qos1(Config, true).
 
+t_redispatch_qos1_no_ack(Config) when is_list(Config) ->
+    test_redispatch_qos1(Config, false).
+
+test_redispatch_qos1(_Config, AckEnabled) ->
+    ok = ensure_config(sticky, AckEnabled),
     Group = <<"group1">>,
     Topic = <<"foo/bar">>,
     ClientId1 = <<"ClientId1">>,
@@ -682,10 +678,169 @@ t_redispatch(_) ->
     emqtt:stop(UsedSubPid2),
     ok.
 
+t_qos1_random_dispatch_if_all_members_are_down(Config) when is_list(Config) ->
+    ok = ensure_config(sticky, true),
+    Group = <<"group1">>,
+    Topic = <<"foo/bar">>,
+    ClientId1 = <<"ClientId1">>,
+    ClientId2 = <<"ClientId2">>,
+    SubOpts = [{clean_start, false}],
+    {ok, ConnPub} = emqtt:start_link([{clientid, <<"pub">>}]),
+    {ok, _} = emqtt:connect(ConnPub),
+
+    {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1} | SubOpts]),
+    {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2} | SubOpts]),
+    {ok, _} = emqtt:connect(ConnPid1),
+    {ok, _} = emqtt:connect(ConnPid2),
+
+    emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
+    emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
+
+    ok = emqtt:stop(ConnPid1),
+    ok = emqtt:stop(ConnPid2),
+
+    [Pid1, Pid2] = emqx_shared_sub:subscribers(Group, Topic),
+    ?assert(is_process_alive(Pid1)),
+    ?assert(is_process_alive(Pid2)),
+
+    {ok, _} = emqtt:publish(ConnPub, Topic, <<"hello11">>, 1),
+    ct:sleep(100),
+    {ok, Msgs1} = gen_server:call(Pid1, get_mqueue),
+    {ok, Msgs2} = gen_server:call(Pid2, get_mqueue),
+    %% assert the message is in mqueue (because socket is closed)
+    ?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2),
+    emqtt:stop(ConnPub),
+    ok.
+
+%% No ack, QoS 2 subscriptions,
+%% client1 receives one message, send pubrec, then suspend
+%% client2 acts normal (auto_ack=true)
+%% Expected behaviour:
+%% the messages sent to client1's inflight and mq are re-dispatched after client1 is down
+t_dispatch_qos2({init, Config}) when is_list(Config) ->
+    emqx_config:put_zone_conf(default, [mqtt, max_inflight], 1),
+    Config;
+t_dispatch_qos2({'end', Config}) when is_list(Config) ->
+    emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0);
+t_dispatch_qos2(Config) when is_list(Config) ->
+    ok = ensure_config(round_robin, _AckEnabled = false),
+    Topic = <<"foo/bar/1">>,
+    ClientId1 = <<"ClientId1">>,
+    ClientId2 = <<"ClientId2">>,
+
+    {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
+    {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
+    {ok, _} = emqtt:connect(ConnPid1),
+    {ok, _} = emqtt:connect(ConnPid2),
+
+    emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}),
+    emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}),
+
+    Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
+    Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
+    Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
+    Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
+    ct:sleep(100),
+
+    ok = sys:suspend(ConnPid1),
+
+    %% One message is inflight
+    ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)),
+    ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)),
+    ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)),
+    ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)),
+
+    MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
+    MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
+    %% assert hello2 > hello1 or hello4 > hello3
+    ?assert(MsgRec2 > MsgRec1),
+
+    sys:resume(ConnPid1),
+    %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false
+    %% so it will never send PUBCOMP, hence EMQX should not attempt to send
+    %% the 4th message yet since max_inflight is 1.
+    MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3),
+    ct:sleep(100),
+    %% no message expected
+    ?assertEqual([], collect_msgs(0)),
+    %% now kill client 1
+    kill_process(ConnPid1),
+    %% client 2 should receive the message
+    MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4),
+    %% assert hello2 > hello1 or hello4 > hello3
+    ?assert(MsgRec4 > MsgRec3),
+    emqtt:stop(ConnPid2),
+    ok.
+
+t_dispatch_qos0({init, Config}) when is_list(Config) ->
+    Config;
+t_dispatch_qos0({'end', Config}) when is_list(Config) ->
+    ok;
+t_dispatch_qos0(Config) when is_list(Config) ->
+    ok = ensure_config(round_robin, _AckEnabled = false),
+    Topic = <<"foo/bar/1">>,
+    ClientId1 = <<"ClientId1">>,
+    ClientId2 = <<"ClientId2">>,
+
+    {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
+    {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
+    {ok, _} = emqtt:connect(ConnPid1),
+    {ok, _} = emqtt:connect(ConnPid2),
+
+    %% subscribe with QoS 0
+    emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 0}),
+    emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 0}),
+
+    %% publish with QoS 2, but should be downgraded to 0 as the subscribers
+    %% subscribe with QoS 0
+    Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
+    Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
+    Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
+    Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
+    ct:sleep(100),
+
+    ok = sys:suspend(ConnPid1),
+
+    ?assertMatch([_], emqx:publish(Message1)),
+    ?assertMatch([_], emqx:publish(Message2)),
+    ?assertMatch([_], emqx:publish(Message3)),
+    ?assertMatch([_], emqx:publish(Message4)),
+
+    MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
+    MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
+    %% assert hello2 > hello1 or hello4 > hello3
+    ?assert(MsgRec2 > MsgRec1),
+
+    kill_process(ConnPid1),
+    %% expect no redispatch
+    ?assertEqual([], collect_msgs(timer:seconds(2))),
+    emqtt:stop(ConnPid2),
+    ok.
+
 %%--------------------------------------------------------------------
 %% help functions
 %%--------------------------------------------------------------------
 
+kill_process(Pid) ->
+    _ = unlink(Pid),
+    _ = monitor(process, Pid),
+    erlang:exit(Pid, kill),
+    receive
+        {'DOWN', _, process, Pid, _} ->
+            ok
+    end.
+
+collect_msgs(Timeout) ->
+    collect_msgs([], Timeout).
+
+collect_msgs(Acc, Timeout) ->
+    receive
+        Msg ->
+            collect_msgs([Msg | Acc], Timeout)
+    after Timeout ->
+        lists:reverse(Acc)
+    end.
+
 ensure_config(Strategy) ->
     ensure_config(Strategy, _AckEnabled = true).
 

apps/emqx_authn/include/emqx_authn.hrl

@@ -38,8 +38,4 @@
 
 -define(RESOURCE_GROUP, <<"emqx_authn">>).
 
--define(WITH_SUCCESSFUL_RENDER(Code),
-    emqx_authn_utils:with_successful_render(?MODULE, fun() -> Code end)
-).
-
 -endif.

apps/emqx_authn/src/emqx_authn.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.6"},
+    {vsn, "0.1.7"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},

apps/emqx_authn/src/emqx_authn_api.erl

@@ -34,7 +34,7 @@
 % Swagger
 
 -define(API_TAGS_GLOBAL, [<<"Authentication">>]).
--define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
+-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
 
 -export([
     api_spec/0,

apps/emqx_authn/src/emqx_authn_user_import_api.erl

@@ -30,7 +30,7 @@
 
 % Swagger
 -define(API_TAGS_GLOBAL, [<<"Authentication">>]).
--define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
+-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
 
 -export([
     api_spec/0,

apps/emqx_authn/src/emqx_authn_utils.erl

@@ -34,8 +34,7 @@
     ensure_apps_started/1,
     cleanup_resources/0,
     make_resource_id/1,
-    without_password/1,
+    without_password/1
-    with_successful_render/2
 ]).
 
 -define(AUTHN_PLACEHOLDERS, [
@@ -112,7 +111,8 @@ parse_sql(Template, ReplaceWith) ->
         Template,
         #{
             replace_with => ReplaceWith,
-            placeholders => ?AUTHN_PLACEHOLDERS
+            placeholders => ?AUTHN_PLACEHOLDERS,
+            strip_double_quote => true
         }
     ).
 
@@ -137,18 +137,6 @@ render_sql_params(ParamList, Credential) ->
         #{return => rawlist, var_trans => fun handle_sql_var/2}
     ).
 
-with_successful_render(Provider, Fun) when is_function(Fun, 0) ->
-    try
-        Fun()
-    catch
-        error:{cannot_get_variable, Name} ->
-            ?TRACE_AUTHN(error, "placeholder_interpolation_failed", #{
-                provider => Provider,
-                placeholder => Name
-            }),
-            ignore
-    end.
-
 %% true
 is_superuser(#{<<"is_superuser">> := <<"true">>}) ->
     #{is_superuser => true};
@@ -230,15 +218,15 @@ without_password(Credential, [Name | Rest]) ->
             without_password(Credential, Rest)
     end.
 
-handle_var({var, Name}, undefined) ->
+handle_var({var, _Name}, undefined) ->
-    error({cannot_get_variable, Name});
+    <<>>;
 handle_var({var, <<"peerhost">>}, PeerHost) ->
     emqx_placeholder:bin(inet:ntoa(PeerHost));
 handle_var(_, Value) ->
     emqx_placeholder:bin(Value).
 
-handle_sql_var({var, Name}, undefined) ->
+handle_sql_var({var, _Name}, undefined) ->
-    error({cannot_get_variable, Name});
+    <<>>;
 handle_sql_var({var, <<"peerhost">>}, PeerHost) ->
     emqx_placeholder:bin(inet:ntoa(PeerHost));
 handle_sql_var(_, Value) ->

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -187,29 +187,25 @@ authenticate(
         request_timeout := RequestTimeout
     } = State
 ) ->
-    ?WITH_SUCCESSFUL_RENDER(
+    Request = generate_request(Credential, State),
-        begin
+    Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
-            Request = generate_request(Credential, State),
+    ?TRACE_AUTHN_PROVIDER("http_response", #{
-            Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
+        request => request_for_log(Credential, State),
-            ?TRACE_AUTHN_PROVIDER("http_response", #{
+        response => response_for_log(Response),
-                request => request_for_log(Credential, State),
+        resource => ResourceId
-                response => response_for_log(Response),
+    }),
-                resource => ResourceId
+    case Response of
-            }),
+        {ok, 204, _Headers} ->
-            case Response of
+            {ok, #{is_superuser => false}};
-                {ok, 204, _Headers} ->
+        {ok, 200, Headers, Body} ->
-                    {ok, #{is_superuser => false}};
+            handle_response(Headers, Body);
-                {ok, 200, Headers, Body} ->
+        {ok, _StatusCode, _Headers} = Response ->
-                    handle_response(Headers, Body);
+            ignore;
-                {ok, _StatusCode, _Headers} = Response ->
+        {ok, _StatusCode, _Headers, _Body} = Response ->
-                    ignore;
+            ignore;
-                {ok, _StatusCode, _Headers, _Body} = Response ->
+        {error, _Reason} ->
-                    ignore;
+            ignore
-                {error, _Reason} ->
+    end.
-                    ignore
-            end
-        end
-    ).
 
 destroy(#{resource_id := ResourceId}) ->
     _ = emqx_resource:remove_local(ResourceId),

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -162,39 +162,35 @@ authenticate(
         resource_id := ResourceId
     } = State
 ) ->
-    ?WITH_SUCCESSFUL_RENDER(
+    Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
-        begin
+    case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
-            Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
+        undefined ->
-            case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
+            ignore;
-                undefined ->
+        {error, Reason} ->
-                    ignore;
+            ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
-                {error, Reason} ->
+                resource => ResourceId,
-                    ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
+                collection => Collection,
+                filter => Filter,
+                reason => Reason
+            }),
+            ignore;
+        Doc ->
+            case check_password(Password, Doc, State) of
+                ok ->
+                    {ok, is_superuser(Doc, State)};
+                {error, {cannot_find_password_hash_field, PasswordHashField}} ->
+                    ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{
                         resource => ResourceId,
                         collection => Collection,
                         filter => Filter,
-                        reason => Reason
+                        document => Doc,
+                        password_hash_field => PasswordHashField
                     }),
                     ignore;
-                Doc ->
+                {error, Reason} ->
-                    case check_password(Password, Doc, State) of
+                    {error, Reason}
-                        ok ->
-                            {ok, is_superuser(Doc, State)};
-                        {error, {cannot_find_password_hash_field, PasswordHashField}} ->
-                            ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{
-                                resource => ResourceId,
-                                collection => Collection,
-                                filter => Filter,
-                                document => Doc,
-                                password_hash_field => PasswordHashField
-                            }),
-                            ignore;
-                        {error, Reason} ->
-                            {error, Reason}
-                    end
             end
-        end
+    end.
-    ).
 
 %%------------------------------------------------------------------------------
 %% Internal functions

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -113,36 +113,32 @@ authenticate(
         password_hash_algorithm := Algorithm
     }
 ) ->
-    ?WITH_SUCCESSFUL_RENDER(
+    Params = emqx_authn_utils:render_sql_params(TmplToken, Credential),
-        begin
+    case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of
-            Params = emqx_authn_utils:render_sql_params(TmplToken, Credential),
+        {ok, _Columns, []} ->
-            case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of
+            ignore;
-                {ok, _Columns, []} ->
+        {ok, Columns, [Row | _]} ->
-                    ignore;
+            Selected = maps:from_list(lists:zip(Columns, Row)),
-                {ok, Columns, [Row | _]} ->
+            case
-                    Selected = maps:from_list(lists:zip(Columns, Row)),
+                emqx_authn_utils:check_password_from_selected_map(
-                    case
+                    Algorithm, Selected, Password
-                        emqx_authn_utils:check_password_from_selected_map(
+                )
-                            Algorithm, Selected, Password
+            of
-                        )
+                ok ->
-                    of
+                    {ok, emqx_authn_utils:is_superuser(Selected)};
-                        ok ->
-                            {ok, emqx_authn_utils:is_superuser(Selected)};
-                        {error, Reason} ->
-                            {error, Reason}
-                    end;
                 {error, Reason} ->
-                    ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{
+                    {error, Reason}
-                        resource => ResourceId,
+            end;
-                        tmpl_token => TmplToken,
+        {error, Reason} ->
-                        params => Params,
+            ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{
-                        timeout => Timeout,
+                resource => ResourceId,
-                        reason => Reason
+                tmpl_token => TmplToken,
-                    }),
+                params => Params,
-                    ignore
+                timeout => Timeout,
-            end
+                reason => Reason
-        end
+            }),
-    ).
+            ignore
+    end.
 
 parse_config(
     #{

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -115,35 +115,31 @@ authenticate(
         password_hash_algorithm := Algorithm
     }
 ) ->
-    ?WITH_SUCCESSFUL_RENDER(
+    Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential),
-        begin
+    case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of
-            Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential),
+        {ok, _Columns, []} ->
-            case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of
+            ignore;
-                {ok, _Columns, []} ->
+        {ok, Columns, [Row | _]} ->
-                    ignore;
+            NColumns = [Name || #column{name = Name} <- Columns],
-                {ok, Columns, [Row | _]} ->
+            Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))),
-                    NColumns = [Name || #column{name = Name} <- Columns],
+            case
-                    Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))),
+                emqx_authn_utils:check_password_from_selected_map(
-                    case
+                    Algorithm, Selected, Password
-                        emqx_authn_utils:check_password_from_selected_map(
+                )
-                            Algorithm, Selected, Password
+            of
-                        )
+                ok ->
-                    of
+                    {ok, emqx_authn_utils:is_superuser(Selected)};
-                        ok ->
-                            {ok, emqx_authn_utils:is_superuser(Selected)};
-                        {error, Reason} ->
-                            {error, Reason}
-                    end;
                 {error, Reason} ->
-                    ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{
+                    {error, Reason}
-                        resource => ResourceId,
+            end;
-                        params => Params,
+        {error, Reason} ->
-                        reason => Reason
+            ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{
-                    }),
+                resource => ResourceId,
-                    ignore
+                params => Params,
-            end
+                reason => Reason
-        end
+            }),
-    ).
+            ignore
+    end.
 
 parse_config(
     #{

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -133,15 +133,14 @@ authenticate(
         password_hash_algorithm := Algorithm
     }
 ) ->
-    ?WITH_SUCCESSFUL_RENDER(
+    NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
-        begin
+    Command = [CommandName, NKey | Fields],
-            NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
+    case emqx_resource:query(ResourceId, {cmd, Command}) of
-            Command = [CommandName, NKey | Fields],
+        {ok, []} ->
-            case emqx_resource:query(ResourceId, {cmd, Command}) of
+            ignore;
-                {ok, []} ->
+        {ok, Values} ->
-                    ignore;
+            case merge(Fields, Values) of
-                {ok, Values} ->
+                Selected when Selected =/= #{} ->
-                    Selected = merge(Fields, Values),
                     case
                         emqx_authn_utils:check_password_from_selected_map(
                             Algorithm, Selected, Password
@@ -149,21 +148,28 @@ authenticate(
                     of
                         ok ->
                             {ok, emqx_authn_utils:is_superuser(Selected)};
-                        {error, _Reason} ->
+                        {error, _Reason} = Error ->
-                            ignore
+                            Error
                     end;
-                {error, Reason} ->
+                _ ->
-                    ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{
+                    ?TRACE_AUTHN_PROVIDER(info, "redis_query_not_matched", #{
                         resource => ResourceId,
                         cmd => Command,
                         keys => NKey,
-                        fields => Fields,
+                        fields => Fields
-                        reason => Reason
                     }),
                     ignore
-            end
+            end;
-        end
+        {error, Reason} ->
-    ).
+            ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{
+                resource => ResourceId,
+                cmd => Command,
+                keys => NKey,
+                fields => Fields,
+                reason => Reason
+            }),
+            ignore
+    end.
 
 %%------------------------------------------------------------------------------
 %% Internal functions

apps/emqx_authn/test/emqx_authn_http_SUITE.erl

@@ -166,6 +166,49 @@ test_user_auth(#{
         ?GLOBAL
     ).
 
+t_no_value_for_placeholder(_Config) ->
+    Handler = fun(Req0, State) ->
+        {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+        #{
+            <<"cert_subject">> := <<"">>,
+            <<"cert_common_name">> := <<"">>
+        } = jiffy:decode(RawBody, [return_maps]),
+        Req = cowboy_req:reply(
+            200,
+            #{<<"content-type">> => <<"application/json">>},
+            jiffy:encode(#{result => allow, is_superuser => false}),
+            Req1
+        ),
+        {ok, Req, State}
+    end,
+
+    SpecificConfgParams = #{
+        <<"method">> => <<"post">>,
+        <<"headers">> => #{<<"content-type">> => <<"application/json">>},
+        <<"body">> => #{
+            <<"cert_subject">> => ?PH_CERT_SUBJECT,
+            <<"cert_common_name">> => ?PH_CERT_CN_NAME
+        }
+    },
+
+    AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams),
+
+    {ok, _} = emqx:update_config(
+        ?PATH,
+        {create_authenticator, ?GLOBAL, AuthConfig}
+    ),
+
+    ok = emqx_authn_http_test_server:set_handler(Handler),
+
+    Credentials = maps:without([cert_subject, cert_common_name], ?CREDENTIALS),
+
+    ?assertMatch({ok, _}, emqx_access_control:authenticate(Credentials)),
+
+    emqx_authn_test_lib:delete_authenticators(
+        [authentication],
+        ?GLOBAL
+    ).
+
 t_destroy(_Config) ->
     AuthConfig = raw_http_auth_config(),
 
@@ -247,27 +290,6 @@ t_update(_Config) ->
         emqx_access_control:authenticate(?CREDENTIALS)
     ).
 
-t_interpolation_error(_Config) ->
-    {ok, _} = emqx:update_config(
-        ?PATH,
-        {create_authenticator, ?GLOBAL, raw_http_auth_config()}
-    ),
-
-    Headers = #{<<"content-type">> => <<"application/json">>},
-    Response = ?SERVER_RESPONSE_JSON(allow),
-
-    ok = emqx_authn_http_test_server:set_handler(
-        fun(Req0, State) ->
-            Req = cowboy_req:reply(200, Headers, Response, Req0),
-            {ok, Req, State}
-        end
-    ),
-
-    ?assertMatch(
-        ?EXCEPTION_DENY,
-        emqx_access_control:authenticate(maps:without([username], ?CREDENTIALS))
-    ).
-
 t_is_superuser(_Config) ->
     Config = raw_http_auth_config(),
     {ok, _} = emqx:update_config(
@@ -431,26 +453,6 @@ samples() ->
             result => {ok, #{is_superuser => false, user_property => #{}}}
         },
 
-        %% simple get request, no username
-        #{
-            handler => fun(Req0, State) ->
-                #{
-                    username := <<"plain">>,
-                    password := <<"plain">>
-                } = cowboy_req:match_qs([username, password], Req0),
-
-                Req = cowboy_req:reply(
-                    200,
-                    #{<<"content-type">> => <<"application/json">>},
-                    jiffy:encode(#{result => allow, is_superuser => false}),
-                    Req0
-                ),
-                {ok, Req, State}
-            end,
-            config_params => #{},
-            result => {ok, #{is_superuser => false, user_property => #{}}}
-        },
-
         %% get request with json body response
         #{
             handler => fun(Req0, State) ->

apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl

@@ -288,20 +288,6 @@ raw_mongo_auth_config() ->
 
 user_seeds() ->
     [
-        #{
-            data => #{
-                username => <<"plain">>,
-                password_hash => <<"plainsalt">>,
-                salt => <<"salt">>,
-                is_superuser => <<"1">>
-            },
-            credentials => #{
-                password => <<"plain">>
-            },
-            config_params => #{},
-            result => {error, not_authorized}
-        },
-
         #{
             data => #{
                 username => <<"plain">>,

apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl

@@ -258,20 +258,6 @@ raw_mysql_auth_config() ->
 
 user_seeds() ->
     [
-        #{
-            data => #{
-                username => "plain",
-                password_hash => "plainsalt",
-                salt => "salt",
-                is_superuser_str => "1"
-            },
-            credentials => #{
-                password => <<"plain">>
-            },
-            config_params => #{},
-            result => {error, not_authorized}
-        },
-
         #{
             data => #{
                 username => "plain",
@@ -332,6 +318,32 @@ user_seeds() ->
             result => {ok, #{is_superuser => true}}
         },
 
+        %% strip double quote support
+        #{
+            data => #{
+                username => "sha256",
+                password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
+                salt => "salt",
+                is_superuser_int => 1
+            },
+            credentials => #{
+                username => <<"sha256">>,
+                password => <<"sha256">>
+            },
+            config_params => #{
+                <<"query">> =>
+                    <<
+                        "SELECT password_hash, salt, is_superuser_int as is_superuser\n"
+                        "                            FROM users where username = \"${username}\" LIMIT 1"
+                    >>,
+                <<"password_hash_algorithm">> => #{
+                    <<"name">> => <<"sha256">>,
+                    <<"salt_position">> => <<"prefix">>
+                }
+            },
+            result => {ok, #{is_superuser => true}}
+        },
+
         #{
             data => #{
                 username => "sha256",

apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl

@@ -320,20 +320,6 @@ raw_pgsql_auth_config() ->
 
 user_seeds() ->
     [
-        #{
-            data => #{
-                username => "plain",
-                password_hash => "plainsalt",
-                salt => "salt",
-                is_superuser_str => "1"
-            },
-            credentials => #{
-                password => <<"plain">>
-            },
-            config_params => #{},
-            result => {error, not_authorized}
-        },
-
         #{
             data => #{
                 username => "plain",
@@ -394,6 +380,32 @@ user_seeds() ->
             result => {ok, #{is_superuser => true}}
         },
 
+        %% strip double quote support
+        #{
+            data => #{
+                username => "sha256",
+                password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
+                salt => "salt",
+                is_superuser_int => 1
+            },
+            credentials => #{
+                username => <<"sha256">>,
+                password => <<"sha256">>
+            },
+            config_params => #{
+                <<"query">> =>
+                    <<
+                        "SELECT password_hash, salt, is_superuser_int as is_superuser\n"
+                        "                            FROM users where username = \"${username}\" LIMIT 1"
+                    >>,
+                <<"password_hash_algorithm">> => #{
+                    <<"name">> => <<"sha256">>,
+                    <<"salt_position">> => <<"prefix">>
+                }
+            },
+            result => {ok, #{is_superuser => true}}
+        },
+
         #{
             data => #{
                 username => "sha256",

apps/emqx_authn/test/emqx_authn_redis_SUITE.erl

@@ -161,11 +161,13 @@ t_authenticate(_Config) ->
         user_seeds()
     ).
 
-test_user_auth(#{
+test_user_auth(
-    credentials := Credentials0,
+    #{
-    config_params := SpecificConfigParams,
+        credentials := Credentials0,
-    result := Result
+        config_params := SpecificConfigParams,
-}) ->
+        result := Result
+    } = Config
+) ->
     AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams),
 
     {ok, _} = emqx:update_config(
@@ -183,14 +185,12 @@ test_user_auth(#{
 
     ?assertEqual(Result, emqx_access_control:authenticate(Credentials)),
 
-    AuthnResult =
+    case maps:get(redis_result, Config, undefined) of
-        case Result of
+        undefined ->
-            {error, _} ->
+            ok;
-                ignore;
+        RedisResult ->
-            Any ->
+            ?assertEqual(RedisResult, emqx_authn_redis:authenticate(Credentials, State))
-                Any
+    end,
-        end,
-    ?assertEqual(AuthnResult, emqx_authn_redis:authenticate(Credentials, State)),
 
     emqx_authn_test_lib:delete_authenticators(
         [authentication],
@@ -292,20 +292,6 @@ raw_redis_auth_config() ->
 
 user_seeds() ->
     [
-        #{
-            data => #{
-                password_hash => <<"plainsalt">>,
-                salt => <<"salt">>,
-                is_superuser => <<"1">>
-            },
-            credentials => #{
-                password => <<"plain">>
-            },
-            key => <<"mqtt_user:plain">>,
-            config_params => #{},
-            result => {error, not_authorized}
-        },
-
         #{
             data => #{
                 password_hash => <<"plainsalt">>,
@@ -478,7 +464,7 @@ user_seeds() ->
                 <<"cmd">> => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
                 <<"password_hash_algorithm">> => #{<<"name">> => <<"bcrypt">>}
             },
-            result => {error, not_authorized}
+            result => {error, bad_username_or_password}
         },
 
         #{
@@ -547,6 +533,23 @@ user_seeds() ->
                 }
             },
             result => {ok, #{is_superuser => true}}
+        },
+
+        %% user not exists
+        #{
+            data => #{
+                password_hash => <<"plainsalt">>,
+                salt => <<"salt">>,
+                is_superuser => <<"1">>
+            },
+            credentials => #{
+                username => <<"not_exists">>,
+                password => <<"plain">>
+            },
+            key => <<"mqtt_user:plain">>,
+            config_params => #{},
+            result => {error, not_authorized},
+            redis_result => ignore
         }
     ].
 

apps/emqx_authz/src/emqx_authz.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [

apps/emqx_authz/src/emqx_authz.erl

@@ -391,14 +391,6 @@ do_authorize(
         Matched ->
             {Matched, Type}
     catch
-        error:{cannot_get_variable, Name} ->
-            emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
-            ?SLOG(warning, #{
-                msg => "placeholder_interpolation_failed",
-                placeholder => Name,
-                authorize_type => Type
-            }),
-            do_authorize(Client, PubSub, Topic, Tail);
         Class:Reason:Stacktrace ->
             emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
             ?SLOG(warning, #{

apps/emqx_authz/src/emqx_authz_api_sources.erl

@@ -223,7 +223,7 @@ sources(get, _) ->
                         ])
                 end;
             (Source, AccIn) ->
-                lists:append(AccIn, [drop_invalid_certs(Source)])
+                lists:append(AccIn, [Source])
         end,
         [],
         get_raw_sources()
@@ -257,7 +257,7 @@ source(get, #{bindings := #{type := Type}}) ->
                     }}
             end;
         [Source] ->
-            {200, drop_invalid_certs(Source)}
+            {200, Source}
     end;
 source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>} = Body}) ->
     update_authz_file(Body);
@@ -511,11 +511,6 @@ update_config(Cmd, Sources) ->
             }}
     end.
 
-drop_invalid_certs(#{<<"ssl">> := SSL} = Source) when SSL =/= undefined ->
-    Source#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
-drop_invalid_certs(Source) ->
-    Source.
-
 parameters_field() ->
     [
         {type,

apps/emqx_authz/src/emqx_authz_http.erl

@@ -45,7 +45,9 @@
     ?PH_PROTONAME,
     ?PH_MOUNTPOINT,
     ?PH_TOPIC,
-    ?PH_ACTION
+    ?PH_ACTION,
+    ?PH_CERT_SUBJECT,
+    ?PH_CERT_CN_NAME
 ]).
 
 description() ->

apps/emqx_authz/src/emqx_authz_mongodb.erl

@@ -40,7 +40,9 @@
 -define(PLACEHOLDERS, [
     ?PH_USERNAME,
     ?PH_CLIENTID,
-    ?PH_PEERHOST
+    ?PH_PEERHOST,
+    ?PH_CERT_CN_NAME,
+    ?PH_CERT_SUBJECT
 ]).
 
 description() ->

apps/emqx_authz/src/emqx_authz_utils.erl

@@ -110,7 +110,8 @@ parse_sql(Template, ReplaceWith, PlaceHolders) ->
         Template,
         #{
             replace_with => ReplaceWith,
-            placeholders => PlaceHolders
+            placeholders => PlaceHolders,
+            strip_double_quote => true
         }
     ).
 
@@ -181,15 +182,15 @@ convert_client_var({dn, DN}) -> {cert_subject, DN};
 convert_client_var({protocol, Proto}) -> {proto_name, Proto};
 convert_client_var(Other) -> Other.
 
-handle_var({var, Name}, undefined) ->
+handle_var({var, _Name}, undefined) ->
-    error({cannot_get_variable, Name});
+    <<>>;
 handle_var({var, <<"peerhost">>}, IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_var(_Name, Value) ->
     emqx_placeholder:bin(Value).
 
-handle_sql_var({var, Name}, undefined) ->
+handle_sql_var({var, _Name}, undefined) ->
-    error({cannot_get_variable, Name});
+    <<>>;
 handle_sql_var({var, <<"peerhost">>}, IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_sql_var(_Name, Value) ->

apps/emqx_authz/test/emqx_authz_SUITE.erl

@@ -19,9 +19,12 @@
 -compile(export_all).
 
 -include("emqx_authz.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -61,10 +64,26 @@ end_per_suite(_Config) ->
     meck:unload(emqx_resource),
     ok.
 
+init_per_testcase(TestCase, Config) when
+    TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
+->
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
+    {ok, _} = emqx:update_config([authorization, deny_action], disconnect),
+    Config;
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
     Config.
 
+end_per_testcase(TestCase, _Config) when
+    TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
+->
+    {ok, _} = emqx:update_config([authorization, deny_action], ignore),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 set_special_configs(emqx_authz) ->
     {ok, _} = emqx:update_config([authorization, cache, enable], false),
     {ok, _} = emqx:update_config([authorization, no_match], deny),
@@ -139,6 +158,15 @@ set_special_configs(_App) ->
             "\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}."
         >>
 }).
+-define(SOURCE7, #{
+    <<"type">> => <<"file">>,
+    <<"enable">> => true,
+    <<"rules">> =>
+        <<
+            "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n"
+            "{deny, all}."
+        >>
+}).
 
 %%------------------------------------------------------------------------------
 %% Testcases
@@ -287,5 +315,87 @@ t_get_enabled_authzs_some_enabled(_Config) ->
     {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4]),
     ?assertEqual([postgresql], emqx_authz:get_enabled_authzs()).
 
+t_subscribe_deny_disconnect_publishes_last_will_testament(_Config) ->
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+    {ok, C} = emqtt:start_link([
+        {username, <<"some_client">>},
+        {will_topic, <<"some_client/lwt">>},
+        {will_payload, <<"should be published">>}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqx:subscribe(<<"some_client/lwt">>),
+    process_flag(trap_exit, true),
+
+    try
+        emqtt:subscribe(C, <<"unauthorized">>),
+        error(should_have_disconnected)
+    catch
+        exit:{{shutdown, tcp_closed}, _} ->
+            ok
+    end,
+
+    receive
+        {deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
+            ok
+    after 2_000 ->
+        error(lwt_not_published)
+    end,
+
+    ok.
+
+t_publish_deny_disconnect_publishes_last_will_testament(_Config) ->
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+    {ok, C} = emqtt:start_link([
+        {username, <<"some_client">>},
+        {will_topic, <<"some_client/lwt">>},
+        {will_payload, <<"should be published">>}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqx:subscribe(<<"some_client/lwt">>),
+    process_flag(trap_exit, true),
+
+    %% disconnect is async
+    Ref = monitor(process, C),
+    emqtt:publish(C, <<"some/topic">>, <<"unauthorized">>),
+    receive
+        {'DOWN', Ref, process, C, _} ->
+            ok
+    after 1_000 ->
+        error(client_should_have_been_disconnected)
+    end,
+    receive
+        {deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
+            ok
+    after 2_000 ->
+        error(lwt_not_published)
+    end,
+
+    ok.
+
+t_publish_last_will_testament_denied_topic(_Config) ->
+    {ok, C} = emqtt:start_link([
+        {will_topic, <<"$SYS/lwt">>},
+        {will_payload, <<"should not be published">>}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqx:subscribe(<<"$SYS/lwt">>),
+    unlink(C),
+    ok = snabbkaffe:start_trace(),
+    {true, {ok, _}} = ?wait_async_action(
+        exit(C, kill),
+        #{?snk_kind := last_will_testament_publish_denied},
+        1_000
+    ),
+    ok = snabbkaffe:stop(),
+
+    receive
+        {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} ->
+            error(lwt_should_not_be_published_to_forbidden_topic)
+    after 1_000 ->
+        ok
+    end,
+
+    ok.
+
 stop_apps(Apps) ->
     lists:foreach(fun application:stop/1, Apps).

apps/emqx_authz/test/emqx_authz_http_SUITE.erl

@@ -22,6 +22,7 @@
 -include("emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_placeholder.hrl").
 
 -define(HTTP_PORT, 33333).
 -define(HTTP_PATH, "/authz/[...]").
@@ -303,7 +304,7 @@ t_json_body(_Config) ->
         emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
     ).
 
-t_form_body(_Config) ->
+t_placeholder_and_body(_Config) ->
     ok = setup_handler_and_config(
         fun(Req0, State) ->
             ?assertEqual(
@@ -321,7 +322,9 @@ t_form_body(_Config) ->
                     <<"proto_name">> := <<"MQTT">>,
                     <<"mountpoint">> := <<"MOUNTPOINT">>,
                     <<"topic">> := <<"t">>,
-                    <<"action">> := <<"publish">>
+                    <<"action">> := <<"publish">>,
+                    <<"CN">> := ?PH_CERT_CN_NAME,
+                    <<"CS">> := ?PH_CERT_SUBJECT
                 },
                 jiffy:decode(PostVars, [return_maps])
             ),
@@ -336,7 +339,9 @@ t_form_body(_Config) ->
                 <<"proto_name">> => <<"${proto_name}">>,
                 <<"mountpoint">> => <<"${mountpoint}">>,
                 <<"topic">> => <<"${topic}">>,
-                <<"action">> => <<"${action}">>
+                <<"action">> => <<"${action}">>,
+                <<"CN">> => ?PH_CERT_CN_NAME,
+                <<"CS">> => ?PH_CERT_SUBJECT
             },
             <<"headers">> => #{<<"content-type">> => <<"application/x-www-form-urlencoded">>}
         }
@@ -349,6 +354,48 @@ t_form_body(_Config) ->
         protocol => <<"MQTT">>,
         mountpoint => <<"MOUNTPOINT">>,
         zone => default,
+        listener => {tcp, default},
+        cn => ?PH_CERT_CN_NAME,
+        dn => ?PH_CERT_SUBJECT
+    },
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
+    ).
+
+t_no_value_for_placeholder(_Config) ->
+    ok = setup_handler_and_config(
+        fun(Req0, State) ->
+            ?assertEqual(
+                <<"/authz/users/">>,
+                cowboy_req:path(Req0)
+            ),
+
+            {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+
+            ?assertMatch(
+                #{
+                    <<"mountpoint">> := <<"[]">>
+                },
+                jiffy:decode(RawBody, [return_maps])
+            ),
+            {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State}
+        end,
+        #{
+            <<"method">> => <<"post">>,
+            <<"body">> => #{
+                <<"mountpoint">> => <<"[${mountpoint}]">>
+            }
+        }
+    ),
+
+    ClientInfo = #{
+        clientid => <<"client id">>,
+        username => <<"user name">>,
+        peerhost => {127, 0, 0, 1},
+        protocol => <<"MQTT">>,
+        zone => default,
         listener => {tcp, default}
     },
 

apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl

@@ -18,8 +18,8 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
 -include("emqx_authz.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
@@ -188,6 +188,46 @@ t_lookups(_Config) ->
         #{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}}
     ),
 
+    ok = emqx_authz_test_lib:test_samples(
+        ClientInfo,
+        [
+            {allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}
+        ]
+    ),
+
+    ByCN = #{
+        <<"CN">> => <<"cn">>,
+        <<"topics">> => [<<"a">>],
+        <<"action">> => <<"all">>,
+        <<"permission">> => <<"allow">>
+    },
+
+    ok = setup_samples([ByCN]),
+    ok = setup_config(
+        #{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}}
+    ),
+
+    ok = emqx_authz_test_lib:test_samples(
+        ClientInfo,
+        [
+            {allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}
+        ]
+    ),
+
+    ByDN = #{
+        <<"DN">> => <<"dn">>,
+        <<"topics">> => [<<"a">>],
+        <<"action">> => <<"all">>,
+        <<"permission">> => <<"allow">>
+    },
+
+    ok = setup_samples([ByDN]),
+    ok = setup_config(
+        #{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}}
+    ),
+
     ok = emqx_authz_test_lib:test_samples(
         ClientInfo,
         [

apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl

@@ -202,6 +202,34 @@ t_lookups(_Config) ->
         }
     ),
 
+    ok = emqx_authz_test_lib:test_samples(
+        ClientInfo,
+        [
+            {allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}
+        ]
+    ),
+
+    %% strip double quote support
+
+    ok = init_table(),
+    ok = q(
+        <<
+            "INSERT INTO acl(clientid, topic, permission, action)"
+            "VALUES(?, ?, ?, ?)"
+        >>,
+        [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
+    ),
+
+    ok = setup_config(
+        #{
+            <<"query">> => <<
+                "SELECT permission, action, topic "
+                "FROM acl WHERE clientid = \"${clientid}\""
+            >>
+        }
+    ),
+
     ok = emqx_authz_test_lib:test_samples(
         ClientInfo,
         [

apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl

@@ -202,6 +202,34 @@ t_lookups(_Config) ->
         }
     ),
 
+    ok = emqx_authz_test_lib:test_samples(
+        ClientInfo,
+        [
+            {allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}
+        ]
+    ),
+
+    %% strip double quote support
+
+    ok = init_table(),
+    ok = insert(
+        <<
+            "INSERT INTO acl(clientid, topic, permission, action)"
+            "VALUES($1, $2, $3, $4)"
+        >>,
+        [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
+    ),
+
+    ok = setup_config(
+        #{
+            <<"query">> => <<
+                "SELECT permission, action, topic "
+                "FROM acl WHERE clientid = \"${clientid}\""
+            >>
+        }
+    ),
+
     ok = emqx_authz_test_lib:test_samples(
         ClientInfo,
         [

apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auto_subscribe, [
     {description, "An OTP application"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_auto_subscribe_app, []}},
     {applications, [

apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl

@@ -44,14 +44,14 @@ schema("/mqtt/auto_subscribe") ->
         'operationId' => auto_subscribe,
         get => #{
             description => ?DESC(list_auto_subscribe_api),
-            tags => [<<"Auto subscribe">>],
+            tags => [<<"Auto Subscribe">>],
             responses => #{
                 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe")
             }
         },
         put => #{
             description => ?DESC(update_auto_subscribe_api),
-            tags => [<<"Auto subscribe">>],
+            tags => [<<"Auto Subscribe">>],
             'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
             responses => #{
                 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -584,10 +584,9 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
 
 format_bridge_info([FirstBridge | _] = Bridges) ->
     Res = maps:remove(node, FirstBridge),
-    NRes = emqx_connector_ssl:drop_invalid_certs(Res),
     NodeStatus = collect_status(Bridges),
     NodeMetrics = collect_metrics(Bridges),
-    NRes#{
+    Res#{
         status => aggregate_status(NodeStatus),
         node_status => NodeStatus,
         metrics => aggregate_metrics(NodeMetrics),

apps/emqx_conf/src/emqx_conf.app.src

@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},

apps/emqx_conf/src/emqx_conf_app.erl

@@ -152,11 +152,17 @@ copy_override_conf_from_core_node() ->
                 _ ->
                     [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
                     #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
-                    Msg = #{
+                    ?SLOG(debug, #{
                         msg => "copy_overide_conf_from_core_node_success",
-                        node => Node
+                        node => Node,
-                    },
+                        cluster_override_conf_file => application:get_env(
-                    ?SLOG(debug, Msg),
+                            emqx, cluster_override_conf_file
+                        ),
+                        local_override_conf_file => application:get_env(
+                            emqx, local_override_conf_file
+                        ),
+                        data_dir => emqx:data_dir()
+                    }),
                     ok = emqx_config:save_to_override_conf(
                         RawOverrideConf,
                         #{override_to => cluster}

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -536,6 +536,15 @@ fields("node") ->
                     desc => ?DESC(node_applications)
                 }
             )},
+        {"etc_dir",
+            sc(
+                string(),
+                #{
+                    desc => ?DESC(node_etc_dir),
+                    'readOnly' => true,
+                    deprecated => {since, "5.0.8"}
+                }
+            )},
         {"cluster_call",
             sc(
                 ?R_REF("cluster_call"),

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -345,7 +345,7 @@ init_worker_options([], Acc) ->
 %% ===================================================================
 %% Schema funcs
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server_raw/1;

apps/emqx_connector/src/emqx_connector_mqtt.erl

@@ -222,20 +222,20 @@ make_forward_confs(undefined) ->
 make_forward_confs(FrowardConf) ->
     FrowardConf.
 
-basic_config(#{
+basic_config(
-    server := Server,
+    #{
-    reconnect_interval := ReconnIntv,
+        server := Server,
-    proto_ver := ProtoVer,
+        reconnect_interval := ReconnIntv,
-    bridge_mode := BridgeMode,
+        proto_ver := ProtoVer,
-    username := User,
+        bridge_mode := BridgeMode,
-    password := Password,
+        clean_start := CleanStart,
-    clean_start := CleanStart,
+        keepalive := KeepAlive,
-    keepalive := KeepAlive,
+        retry_interval := RetryIntv,
-    retry_interval := RetryIntv,
+        max_inflight := MaxInflight,
-    max_inflight := MaxInflight,
+        replayq := ReplayQ,
-    replayq := ReplayQ,
+        ssl := #{enable := EnableSsl} = Ssl
-    ssl := #{enable := EnableSsl} = Ssl
+    } = Conf
-}) ->
+) ->
     #{
         replayq => ReplayQ,
         %% connection opts
@@ -251,8 +251,9 @@ basic_config(#{
         %% non-standard mqtt connection packets will be filtered out by LB.
         %% So let's disable bridge_mode.
         bridge_mode => BridgeMode,
-        username => User,
+        %% should be iolist for emqtt
-        password => Password,
+        username => maps:get(username, Conf, <<>>),
+        password => maps:get(password, Conf, <<>>),
         clean_start => CleanStart,
         keepalive => ms_to_s(KeepAlive),
         retry_interval => RetryIntv,

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -55,7 +55,7 @@ fields(config) ->
         emqx_connector_schema_lib:ssl_fields() ++
         emqx_connector_schema_lib:prepare_statement_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server/1;

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -58,7 +58,7 @@ fields(config) ->
         emqx_connector_schema_lib:ssl_fields() ++
         emqx_connector_schema_lib:prepare_statement_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server/1;

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -97,7 +97,7 @@ fields(sentinel) ->
         redis_fields() ++
         emqx_connector_schema_lib:ssl_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server_raw/1;

apps/emqx_connector/src/emqx_connector_ssl.erl

@@ -18,7 +18,6 @@
 
 -export([
     convert_certs/2,
-    drop_invalid_certs/1,
     clear_certs/2
 ]).
 
@@ -61,28 +60,6 @@ clear_certs(RltvDir, #{ssl := OldSSL} = _Config) ->
 clear_certs(_RltvDir, _) ->
     ok.
 
-drop_invalid_certs(#{<<"connector">> := Connector} = Config) when
-    is_map(Connector)
-->
-    SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
-    NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
-    new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{connector := Connector} = Config) when
-    is_map(Connector)
-->
-    SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
-    NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
-    new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{<<"ssl">> := SSL} = Config) ->
-    NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
-    new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{ssl := SSL} = Config) ->
-    NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
-    new_ssl_config(Config, NewSSL);
-%% for bridges use connector name
-drop_invalid_certs(Config) ->
-    Config.
-
 new_ssl_config(RltvDir, Config, SSL) ->
     case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
         {ok, NewSSL} ->

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl

@@ -55,7 +55,7 @@ fields("connector") ->
             )},
         {server,
             sc(
-                emqx_schema:ip_port(),
+                emqx_schema:host_port(),
                 #{
                     required => true,
                     desc => ?DESC("server")
@@ -177,7 +177,7 @@ fields("ingress") ->
             sc(
                 binary(),
                 #{
-                    default => <<"${payload}">>,
+                    default => undefined,
                     desc => ?DESC("payload")
                 }
             )}
@@ -224,7 +224,7 @@ fields("egress") ->
             sc(
                 binary(),
                 #{
-                    default => <<"${payload}">>,
+                    default => undefined,
                     desc => ?DESC("payload")
                 }
             )}

apps/emqx_dashboard/src/emqx_dashboard.app.src

@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},

apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl

@@ -51,7 +51,7 @@ schema("/error_codes") ->
         get => #{
             security => [],
             description => <<"API Error Codes">>,
-            tags => [<<"Error codes">>],
+            tags => [<<"Error Codes">>],
             responses => #{
                 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code))
             }
@@ -63,7 +63,7 @@ schema("/error_codes/:code") ->
         get => #{
             security => [],
             description => <<"API Error Codes">>,
-            tags => [<<"Error codes">>],
+            tags => [<<"Error Codes">>],
             parameters => [
                 {code,
                     hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) ->
     #{type => string, example => <<"/path/to/file">>};
 typename_to_spec("ip_port()", _Mod) ->
     #{type => string, example => <<"127.0.0.1:80">>};
-typename_to_spec("ip_ports()", _Mod) ->
+typename_to_spec("host_port()", _Mod) ->
-    #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
+    #{type => string, example => <<"example.host.domain:80">>};
 typename_to_spec("url()", _Mod) ->
     #{type => string, example => <<"http://127.0.0.1">>};
 typename_to_spec("connect_timeout()", Mod) ->

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -74,14 +74,6 @@ end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]),
     mria:stop().
 
-set_special_configs(emqx_management) ->
-    Listeners = #{http => #{port => 8081}},
-    Config = #{
-        listeners => Listeners,
-        applications => [#{id => "admin", secret => "public"}]
-    },
-    emqx_config:put([emqx_management], Config),
-    ok;
 set_special_configs(emqx_dashboard) ->
     emqx_dashboard_api_test_helpers:set_default_config(),
     ok;

apps/emqx_exhook/include/emqx_exhook.hrl

@@ -43,6 +43,8 @@
     {'message.dropped', {emqx_exhook_handler, on_message_dropped, []}}
 ]).
 
+-define(SERVER_FORCE_SHUTDOWN_TIMEOUT, 5000).
+
 -endif.
 
 -define(CMD_MOVE_FRONT, front).

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook_api.erl

@@ -41,7 +41,7 @@
 -import(hoconsc, [mk/1, mk/2, ref/1, enum/1, array/1, map/2]).
 -import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]).
 
--define(TAGS, [<<"exhooks">>]).
+-define(TAGS, [<<"ExHook">>]).
 -define(NOT_FOURD, 'NOT_FOUND').
 -define(BAD_REQUEST, 'BAD_REQUEST').
 -define(BAD_RPC, 'BAD_RPC').
@@ -483,16 +483,11 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
 get_raw_config() ->
     RawConfig = emqx:get_raw_config([exhook, servers], []),
     Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
-    Conf = #{<<"servers">> => lists:map(fun drop_invalid_certs/1, RawConfig)},
+    Conf = #{<<"servers">> => RawConfig},
     Options = #{only_fill_defaults => true},
     #{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
     Servers.
 
-drop_invalid_certs(#{<<"ssl">> := SSL} = Conf) when SSL =/= undefined ->
-    Conf#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
-drop_invalid_certs(Conf) ->
-    Conf.
-
 position_example() ->
     #{
         front =>

apps/emqx_exhook/src/emqx_exhook_mgr.erl

@@ -21,6 +21,7 @@
 
 -include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% APIs
 -export([start_link/0]).
@@ -297,7 +298,8 @@ handle_info(refresh_tick, State) ->
 handle_info(_Info, State) ->
     {noreply, State}.
 
-terminate(_Reason, State = #{servers := Servers}) ->
+terminate(Reason, State = #{servers := Servers}) ->
+    _ = unload_exhooks(),
     _ = maps:fold(
         fun(Name, _, AccIn) ->
             do_unload_server(Name, AccIn)
@@ -305,7 +307,7 @@ terminate(_Reason, State = #{servers := Servers}) ->
         State,
         Servers
     ),
-    _ = unload_exhooks(),
+    ?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->

apps/emqx_exhook/src/emqx_exhook_server.erl

@@ -179,13 +179,16 @@ filter(Ls) ->
 
 -spec unload(server()) -> ok.
 unload(#{name := Name, options := ReqOpts, hookspec := HookSpecs}) ->
-    _ = do_deinit(Name, ReqOpts),
     _ = may_unload_hooks(HookSpecs),
+    _ = do_deinit(Name, ReqOpts),
     _ = emqx_exhook_sup:stop_grpc_client_channel(Name),
     ok.
 
 do_deinit(Name, ReqOpts) ->
-    _ = do_call(Name, undefined, 'on_provider_unloaded', #{}, ReqOpts),
+    %% Override the request timeout to deinit grpc server to
+    %% avoid emqx_exhook_mgr force killed by upper supervisor
+    NReqOpts = ReqOpts#{timeout => ?SERVER_FORCE_SHUTDOWN_TIMEOUT},
+    _ = do_call(Name, undefined, 'on_provider_unloaded', #{}, NReqOpts),
     ok.
 
 do_init(ChannName, ReqOpts) ->

apps/emqx_exhook/src/emqx_exhook_sup.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_exhook_sup).
 
+-include("emqx_exhook.hrl").
+
 -behaviour(supervisor).
 
 -export([
@@ -28,11 +30,13 @@
     stop_grpc_client_channel/1
 ]).
 
--define(CHILD(Mod, Type, Args), #{
+-define(DEFAULT_TIMEOUT, 5000).
+
+-define(CHILD(Mod, Type, Args, Timeout), #{
     id => Mod,
     start => {Mod, start_link, Args},
     type => Type,
-    shutdown => 15000
+    shutdown => Timeout
 }).
 
 %%--------------------------------------------------------------------
@@ -45,7 +49,7 @@ start_link() ->
 init([]) ->
     _ = emqx_exhook_metrics:init(),
     _ = emqx_exhook_mgr:init_ref_counter_table(),
-    Mngr = ?CHILD(emqx_exhook_mgr, worker, []),
+    Mngr = ?CHILD(emqx_exhook_mgr, worker, [], force_shutdown_timeout()),
     {ok, {{one_for_one, 10, 100}, [Mngr]}}.
 
 %%--------------------------------------------------------------------
@@ -70,3 +74,9 @@ stop_grpc_client_channel(Name) ->
         _:_:_ ->
             ok
     end.
+
+%% Calculate the maximum timeout, which will help to shutdown the
+%% emqx_exhook_mgr process correctly.
+force_shutdown_timeout() ->
+    Factor = max(3, length(emqx:get_config([exhook, servers])) + 1),
+    Factor * ?SERVER_FORCE_SHUTDOWN_TIMEOUT.

apps/emqx_exhook/test/emqx_exhook_SUITE.erl

@@ -24,6 +24,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_hooks.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(DEFAULT_CLUSTER_NAME_ATOM, emqxcl).
 
@@ -313,6 +314,40 @@ t_cluster_name(_) ->
     ),
     emqx_exhook_mgr:disable(<<"default">>).
 
+t_stop_timeout(_) ->
+    snabbkaffe:start_trace(),
+    meck:new(emqx_exhook_demo_svr, [passthrough, no_history]),
+    meck:expect(
+        emqx_exhook_demo_svr,
+        on_provider_unloaded,
+        fun(Req, Md) ->
+            %% ensure sleep time greater than emqx_exhook_mgr shutdown timeout
+            timer:sleep(20000),
+            meck:passthrough([Req, Md])
+        end
+    ),
+
+    %% stop application
+    application:stop(emqx_exhook),
+    ?block_until(#{?snk_kind := exhook_mgr_terminated}, 20000),
+
+    %% all exhook hooked point should be unloaded
+    Mods = lists:flatten(
+        lists:map(
+            fun({hook, _, Cbs}) ->
+                lists:map(fun({callback, {M, _, _}, _, _}) -> M end, Cbs)
+            end,
+            ets:tab2list(emqx_hooks)
+        )
+    ),
+    ?assertEqual(false, lists:any(fun(M) -> M == emqx_exhook_handler end, Mods)),
+
+    %% ensure started for other tests
+    emqx_common_test_helpers:start_apps([emqx_exhook]),
+
+    snabbkaffe:stop(),
+    meck:unload(emqx_exhook_demo_svr).
+
 %%--------------------------------------------------------------------
 %% Cases Helpers
 %%--------------------------------------------------------------------

apps/emqx_exhook/test/emqx_exhook_demo_svr.erl

@@ -80,7 +80,16 @@ stop() ->
 
 stop(Name) ->
     grpc:stop_server(Name),
-    to_atom_name(Name) ! stop.
+    case whereis(to_atom_name(Name)) of
+        undefined ->
+            ok;
+        Pid ->
+            Ref = erlang:monitor(process, Pid),
+            Pid ! stop,
+            receive
+                {'DOWN', Ref, process, Pid, _Reason} -> ok
+            end
+    end.
 
 take() ->
     to_atom_name(?NAME) ! {take, self()},

apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf

@@ -2,70 +2,76 @@ emqx_gateway_api_authn {
 
     get_authn {
         desc {
-            en: """Get the gateway authentication"""
+            en: """Gets the configuration of the specified gateway authenticator.</br>
-            zh: """获取指定网关认证器"""
+Returns 404 when gateway or authentication is not enabled."""
+            zh: """获取指定网关认证器的配置
+当网关或认证未启用时，返回 404。"""
         }
     }
 
     update_authn {
         desc {
-            en: """Update authentication for the gateway"""
+            en: """Update the configuration of the specified gateway authenticator, or disable the authenticator."""
-            zh: """更新网关认证器"""
+            zh: """更新指定网关认证器的配置，或停用认证器。"""
         }
     }
 
     add_authn {
         desc {
-            en: """Add authentication for the gateway"""
+            en: """Enables the authenticator for client authentication for the specified gateway. <\br>
-            zh: """为指定网关新增认证器"""
+When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br>
+Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT."""
+            zh: """为指定网关开启认证器实现客户端认证的功能。<\br>
+当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。<\br>
+注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。"""
         }
     }
 
     delete_authn {
         desc {
-            en: """Remove the gateway authentication"""
+            en: """Delete the authenticator of the specified gateway."""
-            zh: """删除指定网关的认证器"""
+            zh: """删除指定网关的认证器。"""
         }
     }
 
     list_users {
         desc {
-            en: """Get the users for the authentication"""
+            en: """Get the users for the authenticator (only supported by <code>built_in_database</code>)."""
             zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     add_user {
         desc {
-            en: """Add user for the authentication"""
+            en: """Add user for the authenticator (only supports built_in_database)."""
             zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     get_user {
         desc {
-            en: """Get user info from the gateway authentication"""
+            en: """Get user info from the gateway authenticator (only supports built_in_database)"""
             zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     update_user {
         desc {
-            en: """Update the user info for the gateway authentication"""
+            en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
             zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     delete_user {
         desc {
-            en: """Delete the user for the gateway authentication"""
+            en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
             zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     import_users {
         desc {
-            en: """Import users into the gateway authentication"""
+            en: """Import users into the gateway authenticator (only supports built_in_database)"""
             zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
@@ -79,8 +85,8 @@ emqx_gateway_api_authn {
 
     like_user_id {
         desc {
-            en: """Fuzzy search by user_id (username or clientid)"""
+            en: """Fuzzy search using user ID (username or clientid), only supports search by substring."""
-            zh: """用户 ID （username 或 clientid）模糊搜索"""
+            zh: """使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。"""
         }
     }
 
@@ -90,5 +96,4 @@ emqx_gateway_api_authn {
             zh: """是否是超级用户"""
         }
     }
-
 }

apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf

@@ -2,15 +2,17 @@ emqx_gateway_api {
 
     list_gateway {
         desc {
-            en: """Get gateway list"""
+            en: """This API returns an overview info for the specified or all gateways.
-            zh: """获取网关列表"""
+including current running status, number of connections, listener status, etc."""
+            zh: """该接口会返回指定或所有网关的概览状态，
+包括当前状态、连接数、监听器状态等。"""
         }
     }
 
     enable_gateway {
         desc {
-            en: """Enable a gateway"""
+            en: """Enable a gateway by confs."""
-            zh: """启用某网关"""
+            zh: """使用配置启动某一网关。"""
         }
     }
 
@@ -23,15 +25,17 @@ emqx_gateway_api {
 
     delete_gateway {
         desc {
-            en: """Delete/Unload the gateway"""
+            en: """Unload the specified gateway"""
-            zh: """删除/禁用某网关"""
+            zh: """停用指定网关"""
         }
     }
 
     update_gateway {
         desc {
-            en: """Update the gateway configurations/status"""
+            en: """Update the gateway basic configurations and running status.</br>
-            zh: """更新网关配置或启用状态"""
+Note: The Authentication and Listener configurations should be updated by other special APIs. """
+            zh: """更新指定网关的基础配置、和启用的状态。</br>
+注：认证、和监听器的配置更新需参考对应的 API 接口。"""
         }
     }
 
@@ -42,13 +46,33 @@ emqx_gateway_api {
         }
     }
 
+    gateway_name_in_qs {
+        desc {
+            en: """Gateway Name.</br>
+It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`
+"""
+            zh: """网关名称.</br>
+可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`
+"""
+        }
+    }
+
     gateway_status {
         desc {
-            en: """Gateway Status"""
+            en: """Gateway status"""
             zh: """网关启用状态"""
         }
     }
 
+    gateway_status_in_qs {
+        desc {
+            en: """Filter gateways by status.</br>
+It is enum with `running`, `stopped`, `unloaded`"""
+            zh: """通过网关状态筛选</br>
+可选值为 `running`、`stopped`、`unloaded`"""
+        }
+    }
+
     gateway_created_at {
         desc {
             en: """The Gateway created datetime"""

apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf

@@ -2,105 +2,109 @@ emqx_gateway_api_listeners {
 
     list_listeners {
         desc {
-            en: """Get the gateway listeners"""
+            en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster."""
-            zh: """获取网关监听器列表"""
+            zh: """获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。"""
         }
     }
 
     add_listener {
         desc {
-            en: """Create the gateway listener"""
+            en: """Create the gateway listener.</br>
-            zh: """为指定网关添加监听器"""
+Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`."""
+            zh: """为指定网关添加监听器。</br>
+注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。"""
         }
     }
 
     get_listener {
         desc {
-            en: """Get the gateway listener configurations"""
+            en: """Get the gateway listener configs"""
-            zh: """获取指定监听器信息"""
+            zh: """获取指定网关监听器的配置。"""
         }
     }
 
     delete_listener {
         desc {
-            en: """Delete the gateway listener"""
+            en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected."""
-            zh: """删除监听器"""
+            zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。"""
         }
     }
 
     update_listener {
         desc {
-            en: """Update the gateway listener"""
+            en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected."""
-            zh: """更新监听器"""
+            zh: """更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。"""
         }
     }
 
     get_listener_authn {
         desc {
-            en: """Get the listener's authentication info"""
+            en: """Get the listener's authenticator configs."""
-            zh: """获取监听器的认证器信息"""
+            zh: """获取监听器的认证器配置。"""
         }
     }
 
     add_listener_authn {
         desc {
-            en: """Add authentication for the listener"""
+            en: """Enable authenticator for specified listener for client authentication.</br>
-            zh: """为指定监听器添加认证器"""
+When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication."""
+            zh: """为指定监听器开启认证器以实现客户端认证的能力。</br>
+当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。"""
         }
     }
 
     update_listener_authn {
         desc {
-            en: """Update authentication for the listener"""
+            en: """Update authenticator configs for the listener, or disable/enable it."""
-            zh: """更新指定监听上的认证器配置"""
+            zh: """更新指定监听器的认证器配置，或停用/启用该认证器。"""
         }
     }
 
     delete_listener_authn {
         desc {
-            en: """Remove authentication for the listener"""
+            en: """Remove authenticator for the listener."""
-            zh: """为指定监听器移除认证器"""
+            zh: """移除指定监听器的认证器。"""
         }
     }
 
     list_users {
         desc {
-            en: """Get the users for the authentication"""
+            en: """Get the users for the authenticator (only supported by <code>built_in_database</code>)"""
             zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     add_user {
         desc {
-            en: """Add user for the authentication"""
+            en: """Add user for the authenticator (only supports built_in_database)"""
             zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     get_user {
         desc {
-            en: """Get user info from the gateway authentication"""
+            en: """Get user info from the gateway authenticator (only supports built_in_database)"""
             zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     update_user {
         desc {
-            en: """Update the user info for the gateway authentication"""
+            en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
             zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     delete_user {
         desc {
-            en: """Delete the user for the gateway authentication"""
+            en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
             zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
 
     import_users {
         desc {
-            en: """Import users into the gateway authentication"""
+            en: """Import users into the gateway authenticator (only supports built_in_database)"""
             zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
         }
     }
@@ -139,5 +143,4 @@ emqx_gateway_api_listeners {
             zh: """当前连接数"""
         }
     }
-
 }

apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl

@@ -19,6 +19,7 @@
 
 -include_lib("emqx/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% API
 -export([
@@ -51,6 +52,9 @@
 %% Internal callback
 -export([wakeup_from_hib/2, recvloop/2]).
 
+%% for channel module
+-export([keepalive_stats/1]).
+
 -record(state, {
     %% TCP/SSL/UDP/DTLS Wrapped Socket
     socket :: {esockd_transport, esockd:socket()} | {udp, _, _},
@@ -240,6 +244,11 @@ esockd_send(Data, #state{
 esockd_send(Data, #state{socket = {esockd_transport, Sock}}) ->
     esockd_transport:async_send(Sock, Data).
 
+keepalive_stats(recv) ->
+    emqx_pd:get_counter(recv_pkt);
+keepalive_stats(send) ->
+    emqx_pd:get_counter(send_pkt).
+
 is_datadram_socket({esockd_transport, _}) -> false;
 is_datadram_socket({udp, _, _}) -> true.
 
@@ -568,9 +577,15 @@ terminate(
         channel = Channel
     }
 ) ->
-    ?SLOG(debug, #{msg => "conn_process_terminated", reason => Reason}),
     _ = ChannMod:terminate(Reason, Channel),
     _ = close_socket(State),
+    ClientId =
+        try ChannMod:info(clientid, Channel) of
+            Id -> Id
+        catch
+            _:_ -> undefined
+        end,
+    ?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}),
     exit(Reason).
 
 %%--------------------------------------------------------------------
@@ -635,28 +650,22 @@ handle_timeout(
     Keepalive,
     State = #state{
         chann_mod = ChannMod,
-        socket = Socket,
         channel = Channel
     }
 ) when
     Keepalive == keepalive;
     Keepalive == keepalive_send
 ->
-    Stat =
+    StatVal =
         case Keepalive of
-            keepalive -> recv_oct;
+            keepalive -> keepalive_stats(recv);
-            keepalive_send -> send_oct
+            keepalive_send -> keepalive_stats(send)
         end,
     case ChannMod:info(conn_state, Channel) of
         disconnected ->
             {ok, State};
         _ ->
-            case esockd_getstat(Socket, [Stat]) of
+            handle_timeout(TRef, {Keepalive, StatVal}, State)
-                {ok, [{Stat, RecvOct}]} ->
-                    handle_timeout(TRef, {Keepalive, RecvOct}, State);
-                {error, Reason} ->
-                    handle_info({sock_error, Reason}, State)
-            end
     end;
 handle_timeout(
     _TRef,

apps/emqx_gateway/src/coap/emqx_coap_api.erl

@@ -48,8 +48,9 @@ schema(?PREFIX ++ "/request") ->
     #{
         operationId => request,
         post => #{
-            tags => [<<"CoAP">>],
+            tags => [<<"Gateways">>],
             desc => ?DESC(send_coap_request),
+            summary => <<"Send a Request to a Client">>,
             parameters => request_parameters(),
             requestBody => request_body(),
             responses => #{

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

apps/emqx_gateway/src/emqx_gateway_api.erl

@@ -164,7 +164,9 @@ schema("/gateways") ->
         'operationId' => gateway,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_gateway),
+                summary => <<"List All Gateways">>,
                 parameters => params_gateway_status_in_qs(),
                 responses =>
                     ?STANDARD_RESP(
@@ -178,7 +180,9 @@ schema("/gateways") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(enable_gateway),
+                summary => <<"Enable a Gateway">>,
                 %% TODO: distinguish create & response swagger schema
                 'requestBody' => schema_gateways_conf(),
                 responses =>
@@ -190,21 +194,27 @@ schema("/gateways/:name") ->
         'operationId' => gateway_insta,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_gateway),
+                summary => <<"Get the Gateway">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(#{200 => schema_gateways_conf()})
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_gateway),
+                summary => <<"Unload the gateway">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Deleted">>})
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_gateway),
+                summary => <<"Update the gateway confs">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => schema_update_gateways_conf(),
                 responses =>
@@ -224,8 +234,8 @@ params_gateway_name_in_path() ->
                 binary(),
                 #{
                     in => path,
-                    desc => ?DESC(gateway_name),
+                    desc => ?DESC(gateway_name_in_qs),
-                    example => <<"">>
+                    example => <<"stomp">>
                 }
             )}
     ].
@@ -239,7 +249,7 @@ params_gateway_status_in_qs() ->
                 #{
                     in => query,
                     required => false,
-                    desc => ?DESC(gateway_status),
+                    desc => ?DESC(gateway_status_in_qs),
                     example => <<"">>
                 }
             )}

apps/emqx_gateway/src/emqx_gateway_api_authn.erl

@@ -75,6 +75,7 @@ authn(get, #{bindings := #{name := Name0}}) ->
             Authn -> {200, Authn}
         catch
             error:{config_not_found, _} ->
+                %% FIXME: should return 404?
                 {204}
         end
     end);
@@ -181,19 +182,23 @@ schema("/gateways/:name/authentication") ->
         'operationId' => authn,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_authn),
+                summary => <<"Get Authenticator Configuration">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(
                         #{
                             200 => schema_authn(),
-                            204 => <<"Authentication does not initiated">>
+                            204 => <<"Authenticator doesn't initiated">>
                         }
                     )
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_authn),
+                summary => <<"Update Authenticator Configuration">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => schema_authn(),
                 responses =>
@@ -201,7 +206,9 @@ schema("/gateways/:name/authentication") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_authn),
+                summary => <<"Create an Authenticator for a Gateway">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => schema_authn(),
                 responses =>
@@ -209,7 +216,9 @@ schema("/gateways/:name/authentication") ->
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_authn),
+                summary => <<"Delete the Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Deleted">>})
@@ -220,7 +229,9 @@ schema("/gateways/:name/authentication/users") ->
         'operationId' => users,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_users),
+                summary => <<"List users for a Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_paging_in_qs() ++
                     params_fuzzy_in_qs(),
@@ -236,7 +247,9 @@ schema("/gateways/:name/authentication/users") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_user),
+                summary => <<"Add User for a Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                     ref(emqx_authn_api, request_user_create),
@@ -258,7 +271,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
         'operationId' => users_insta,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_user),
+                summary => <<"Get User Info for a Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 responses =>
@@ -273,7 +288,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_user),
+                summary => <<"Update User Info for a Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -292,7 +309,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_user),
+                summary => <<"Delete User for a Gateway Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 responses =>
@@ -311,8 +330,8 @@ params_gateway_name_in_path() ->
                 binary(),
                 #{
                     in => path,
-                    desc => ?DESC(emqx_gateway_api, gateway_name),
+                    desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
-                    example => <<"">>
+                    example => <<"stomp">>
                 }
             )}
     ].
@@ -325,7 +344,7 @@ params_userid_in_path() ->
                 #{
                     in => path,
                     desc => ?DESC(user_id),
-                    example => <<"">>
+                    example => <<"test_username">>
                 }
             )}
     ].
@@ -343,7 +362,7 @@ params_fuzzy_in_qs() ->
                     in => query,
                     required => false,
                     desc => ?DESC(like_user_id),
-                    example => <<"username">>
+                    example => <<"test_">>
                 }
             )},
         {is_superuser,

apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl

@@ -122,7 +122,9 @@ schema("/gateways/:name/authentication/import_users") ->
         'operationId' => import_users,
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(emqx_gateway_api_authn, import_users),
+                summary => <<"Import Users">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => emqx_dashboard_swagger:file_schema(filename),
                 responses =>
@@ -134,7 +136,9 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
         'operationId' => import_listener_users,
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(emqx_gateway_api_listeners, import_users),
+                summary => <<"Import Users">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:file_schema(filename),
@@ -144,6 +148,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
     };
 schema(Path) ->
     emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
+
 %%--------------------------------------------------------------------
 %% params defines
 %%--------------------------------------------------------------------
@@ -155,7 +160,7 @@ params_gateway_name_in_path() ->
                 binary(),
                 #{
                     in => path,
-                    desc => ?DESC(emqx_gateway_api, gateway_name),
+                    desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
                     example => <<"stomp">>
                 }
             )}

apps/emqx_gateway/src/emqx_gateway_api_clients.erl

@@ -467,7 +467,9 @@ schema("/gateways/:name/clients") ->
         'operationId' => clients,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_clients),
+                summary => <<"List Gateway's Clients">>,
                 parameters => params_client_query(),
                 responses =>
                     ?STANDARD_RESP(#{200 => schema_client_list()})
@@ -478,14 +480,18 @@ schema("/gateways/:name/clients/:clientid") ->
         'operationId' => clients_insta,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_client),
+                summary => <<"Get Client Info">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{200 => schema_client()})
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(kick_client),
+                summary => <<"Kick out Client">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Kicked">>})
@@ -496,7 +502,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
         'operationId' => subscriptions,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_subscriptions),
+                summary => <<"List Client's Subscription">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(
@@ -510,7 +518,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_subscription),
+                summary => <<"Add Subscription for Client">>,
                 parameters => params_client_insta(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                     ref(subscription),
@@ -532,7 +542,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") ->
         'operationId' => subscriptions,
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_subscription),
+                summary => <<"Delete Client's Subscription">>,
                 parameters => params_topic_name_in_path() ++ params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Unsubscribed">>})

apps/emqx_gateway/src/emqx_gateway_api_listeners.erl

@@ -358,7 +358,9 @@ schema("/gateways/:name/listeners") ->
         'operationId' => listeners,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_listeners),
+                summary => <<"List All Listeners">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(
@@ -372,7 +374,9 @@ schema("/gateways/:name/listeners") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_listener),
+                summary => <<"Add a Listener">>,
                 parameters => params_gateway_name_in_path(),
                 %% XXX: How to distinguish the different listener supported by
                 %% different types of gateways?
@@ -396,7 +400,9 @@ schema("/gateways/:name/listeners/:id") ->
         'operationId' => listeners_insta,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_listener),
+                summary => <<"Get the Listener Configs">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -411,7 +417,9 @@ schema("/gateways/:name/listeners/:id") ->
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_listener),
+                summary => <<"Delete the Listener">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -419,7 +427,9 @@ schema("/gateways/:name/listeners/:id") ->
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_listener),
+                summary => <<"Update the Listener Configs">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -442,7 +452,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
         'operationId' => listeners_insta_authn,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_listener_authn),
+                summary => <<"Get the Listener's Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -455,7 +467,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_listener_authn),
+                summary => <<"Create an Authenticator for a Listener">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => schema_authn(),
@@ -464,7 +478,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_listener_authn),
+                summary => <<"Update the Listener Authenticator configs">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => schema_authn(),
@@ -473,7 +489,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_listener_authn),
+                summary => <<"Delete the Listener's Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -485,7 +503,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
         'operationId' => users,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(list_users),
+                summary => <<"List Authenticator's Users">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_paging_in_qs(),
@@ -501,7 +521,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
             },
         post =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(add_user),
+                summary => <<"Add User for an Authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -524,7 +546,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
         'operationId' => users_insta,
         get =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(get_user),
+                summary => <<"Get User Info">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -540,7 +564,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
             },
         put =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(update_user),
+                summary => <<"Update User Info">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -560,7 +586,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
             },
         delete =>
             #{
+                tags => [<<"Gateways">>],
                 desc => ?DESC(delete_user),
+                summary => <<"Delete User">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -570,6 +598,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
     };
 schema(Path) ->
     emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
+
 %%--------------------------------------------------------------------
 %% params defines
 
@@ -580,8 +609,8 @@ params_gateway_name_in_path() ->
                 binary(),
                 #{
                     in => path,
-                    desc => ?DESC(emqx_gateway_api, gateway_name),
+                    desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
-                    example => <<"">>
+                    example => <<"stomp">>
                 }
             )}
     ].

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -28,7 +28,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
 
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
 -type duration() :: non_neg_integer().
 -type duration_s() :: non_neg_integer().
 -type bytesize() :: pos_integer().

apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl

@@ -78,7 +78,8 @@
 
 -define(TIMER_TABLE, #{
     alive_timer => keepalive,
-    force_timer => force_close
+    force_timer => force_close,
+    idle_timer => force_close_idle
 }).
 
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@@ -151,14 +152,17 @@ init(
     Ctx = maps:get(ctx, Options),
     GRpcChann = maps:get(handler, Options),
     PoolName = maps:get(pool_name, Options),
-    NConnInfo = default_conninfo(ConnInfo),
+    IdleTimeout = emqx_gateway_utils:idle_timeout(Options),
+
+    NConnInfo = default_conninfo(ConnInfo#{idle_timeout => IdleTimeout}),
     ListenerId =
         case maps:get(listener, Options, undefined) of
             undefined -> undefined;
             {GwName, Type, LisName} -> emqx_gateway_utils:listener_id(GwName, Type, LisName)
         end,
+
     EnableAuthn = maps:get(enable_authn, Options, true),
-    DefaultClientInfo = default_clientinfo(ConnInfo),
+    DefaultClientInfo = default_clientinfo(NConnInfo),
     ClientInfo = DefaultClientInfo#{
         listener => ListenerId,
         enable_authn => EnableAuthn
@@ -183,7 +187,9 @@ init(
                 }
             )
     },
-    try_dispatch(on_socket_created, wrap(Req), Channel).
+    start_idle_checking_timer(
+        try_dispatch(on_socket_created, wrap(Req), Channel)
+    ).
 
 %% @private
 peercert(NoSsl, ConnInfo) when
@@ -217,6 +223,12 @@ socktype(dtls) -> 'DTLS'.
 address({Host, Port}) ->
     #{host => inet:ntoa(Host), port => Port}.
 
+%% avoid udp connection process leak
+start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) ->
+    ensure_timer(idle_timer, Channel);
+start_idle_checking_timer(Channel) ->
+    Channel.
+
 %%--------------------------------------------------------------------
 %% Handle incoming packet
 %%--------------------------------------------------------------------
@@ -285,10 +297,15 @@ handle_timeout(
             {ok, reset_timer(alive_timer, NChannel)};
         {error, timeout} ->
             Req = #{type => 'KEEPALIVE'},
-            {ok, try_dispatch(on_timer_timeout, wrap(Req), Channel)}
+            NChannel = remove_timer_ref(alive_timer, Channel),
+            %% close connection if keepalive timeout
+            Replies = [{event, disconnected}, {close, keepalive_timeout}],
+            {ok, Replies, try_dispatch(on_timer_timeout, wrap(Req), NChannel)}
     end;
 handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) ->
     {shutdown, {error, {force_close, Reason}}, Channel};
+handle_timeout(_TRef, force_close_idle, Channel) ->
+    {shutdown, idle_timeout, Channel};
 handle_timeout(_TRef, Msg, Channel) ->
     ?SLOG(warning, #{
         msg => "unexpected_timeout_signal",
@@ -390,7 +407,7 @@ handle_call(
     NConnInfo = ConnInfo#{keepalive => Interval},
     NClientInfo = ClientInfo#{keepalive => Interval},
     NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo},
-    {reply, ok, ensure_keepalive(NChannel)};
+    {reply, ok, [{event, updated}], ensure_keepalive(cancel_timer(idle_timer, NChannel))};
 handle_call(
     {subscribe_from_client, TopicFilter, Qos},
     _From,
@@ -405,21 +422,21 @@ handle_call(
             {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel};
         _ ->
             {ok, _, NChannel} = do_subscribe([{TopicFilter, #{qos => Qos}}], Channel),
-            {reply, ok, NChannel}
+            {reply, ok, [{event, updated}], NChannel}
     end;
 handle_call({subscribe, Topic, SubOpts}, _From, Channel) ->
     {ok, [{NTopicFilter, NSubOpts}], NChannel} = do_subscribe([{Topic, SubOpts}], Channel),
-    {reply, {ok, {NTopicFilter, NSubOpts}}, NChannel};
+    {reply, {ok, {NTopicFilter, NSubOpts}}, [{event, updated}], NChannel};
 handle_call(
     {unsubscribe_from_client, TopicFilter},
     _From,
     Channel = #channel{conn_state = connected}
 ) ->
     {ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel),
-    {reply, ok, NChannel};
+    {reply, ok, [{event, updated}], NChannel};
 handle_call({unsubscribe, Topic}, _From, Channel) ->
     {ok, NChannel} = do_unsubscribe([Topic], Channel),
-    {reply, ok, NChannel};
+    {reply, ok, [{event, update}], NChannel};
 handle_call(subscriptions, _From, Channel = #channel{subscriptions = Subs}) ->
     {reply, {ok, maps:to_list(Subs)}, Channel};
 handle_call(
@@ -446,7 +463,7 @@ handle_call(
             {reply, ok, Channel}
     end;
 handle_call(kick, _From, Channel) ->
-    {shutdown, kicked, ok, ensure_disconnected(kicked, Channel)};
+    {reply, ok, [{event, disconnected}, {close, kicked}], Channel};
 handle_call(discard, _From, Channel) ->
     {shutdown, discarded, ok, Channel};
 handle_call(Req, _From, Channel) ->
@@ -648,7 +665,8 @@ ensure_keepalive(Channel = #channel{clientinfo = ClientInfo}) ->
 ensure_keepalive_timer(Interval, Channel) when Interval =< 0 ->
     Channel;
 ensure_keepalive_timer(Interval, Channel) ->
-    Keepalive = emqx_keepalive:init(timer:seconds(Interval)),
+    StatVal = emqx_gateway_conn:keepalive_stats(recv),
+    Keepalive = emqx_keepalive:init(StatVal, timer:seconds(Interval)),
     ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}).
 
 ensure_timer(Name, Channel = #channel{timers = Timers}) ->
@@ -666,11 +684,17 @@ ensure_timer(Name, Time, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = Timers#{Name => TRef}}.
 
 reset_timer(Name, Channel) ->
-    ensure_timer(Name, clean_timer(Name, Channel)).
+    ensure_timer(Name, remove_timer_ref(Name, Channel)).
 
-clean_timer(Name, Channel = #channel{timers = Timers}) ->
+cancel_timer(Name, Channel = #channel{timers = Timers}) ->
+    emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)),
+    remove_timer_ref(Name, Channel).
+
+remove_timer_ref(Name, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = maps:remove(Name, Timers)}.
 
+interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) ->
+    IdleTimeout;
 interval(force_timer, _) ->
     15000;
 interval(alive_timer, #channel{keepalive = Keepalive}) ->
@@ -725,7 +749,7 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) ->
 default_conninfo(ConnInfo) ->
     ConnInfo#{
         clean_start => true,
-        clientid => undefined,
+        clientid => anonymous_clientid(),
         username => undefined,
         conn_props => #{},
         connected => true,
@@ -739,14 +763,15 @@ default_conninfo(ConnInfo) ->
 
 default_clientinfo(#{
     peername := {PeerHost, _},
-    sockname := {_, SockPort}
+    sockname := {_, SockPort},
+    clientid := ClientId
 }) ->
     #{
         zone => default,
         protocol => exproto,
         peerhost => PeerHost,
         sockport => SockPort,
-        clientid => undefined,
+        clientid => ClientId,
         username => undefined,
         is_bridge => false,
         is_superuser => false,
@@ -764,3 +789,6 @@ proto_name_to_protocol(<<>>) ->
     exproto;
 proto_name_to_protocol(ProtoName) when is_binary(ProtoName) ->
     binary_to_atom(ProtoName).
+
+anonymous_clientid() ->
+    iolist_to_binary(["exproto-", emqx_misc:gen_id()]).

apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl

@@ -56,12 +56,19 @@ start_link(Pool, Id) ->
         []
     ).
 
+-spec async_call(atom(), map(), map()) -> ok.
 async_call(
     FunName,
     Req = #{conn := Conn},
     Options = #{pool_name := PoolName}
 ) ->
-    cast(pick(PoolName, Conn), {rpc, FunName, Req, Options, self()}).
+    case pick(PoolName, Conn) of
+        false ->
+            reply(self(), FunName, {error, no_available_grpc_client});
+        Pid when is_pid(Pid) ->
+            cast(Pid, {rpc, FunName, Req, Options, self()})
+    end,
+    ok.
 
 %%--------------------------------------------------------------------
 %% cast, pick
@@ -72,6 +79,7 @@ async_call(
 cast(Deliver, Msg) ->
     gen_server:cast(Deliver, Msg).
 
+-spec pick(term(), term()) -> pid() | false.
 pick(PoolName, Conn) ->
     gproc_pool:pick_worker(PoolName, Conn).
 

apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl

@@ -45,8 +45,9 @@ schema(?PATH("/lookup")) ->
     #{
         'operationId' => lookup,
         get => #{
-            tags => [<<"LwM2M">>],
+            tags => [<<"Gateways">>],
             desc => ?DESC(lookup_resource),
+            summary => <<"List Client's Resources">>,
             parameters => [
                 {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
                 {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@@ -69,8 +70,9 @@ schema(?PATH("/observe")) ->
     #{
         'operationId' => observe,
         post => #{
-            tags => [<<"LwM2M">>],
+            tags => [<<"Gateways">>],
             desc => ?DESC(observe_resource),
+            summary => <<"Observe a Resource">>,
             parameters => [
                 {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
                 {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@@ -87,8 +89,9 @@ schema(?PATH("/read")) ->
     #{
         'operationId' => read,
         post => #{
-            tags => [<<"LwM2M">>],
+            tags => [<<"Gateways">>],
             desc => ?DESC(read_resource),
+            summary => <<"Read Value from a Resource Path">>,
             parameters => [
                 {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
                 {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}
@@ -104,7 +107,8 @@ schema(?PATH("/write")) ->
         'operationId' => write,
         post => #{
             desc => ?DESC(write_resource),
-            tags => [<<"LwM2M">>],
+            tags => [<<"Gateways">>],
+            summary => <<"Write a Value to Resource Path">>,
             parameters => [
                 {clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
                 {path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},

apps/emqx_gateway/test/emqx_exproto_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx_hooks.hrl").
+-include_lib("eunit/include/eunit.hrl").
 
 -import(
     emqx_exproto_echo_svr,
@@ -38,6 +39,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(TCPOPTS, [binary, {active, false}]).
 -define(DTLSOPTS, [binary, {active, false}, {protocol, dtls}]).
@@ -62,6 +64,9 @@
 all() ->
     [{group, Name} || Name <- metrics()].
 
+suite() ->
+    [{timetrap, {seconds, 30}}].
+
 groups() ->
     Cases = emqx_common_test_helpers:all(?MODULE),
     [{Name, Cases} || Name <- metrics()].
@@ -87,6 +92,7 @@ set_special_cfg(emqx_gateway) ->
         [gateway, exproto],
         #{
             server => #{bind => 9100},
+            idle_timeout => 5000,
             handler => #{address => "http://127.0.0.1:9001"},
             listeners => listener_confs(LisType)
         }
@@ -223,14 +229,16 @@ t_acl_deny(Cfg) ->
     close(Sock).
 
 t_keepalive_timeout(Cfg) ->
+    ok = snabbkaffe:start_trace(),
     SockType = proplists:get_value(listener_type, Cfg),
     Sock = open(SockType),
 
+    ClientId1 = <<"keepalive_test_client1">>,
     Client = #{
         proto_name => <<"demo">>,
         proto_ver => <<"v0.1">>,
-        clientid => <<"test_client_1">>,
+        clientid => ClientId1,
-        keepalive => 2
+        keepalive => 5
     },
     Password = <<"123456">>,
 
@@ -238,16 +246,42 @@ t_keepalive_timeout(Cfg) ->
     ConnAckBin = frame_connack(0),
 
     send(Sock, ConnBin),
-    {ok, ConnAckBin} = recv(Sock, 5000),
+    {ok, ConnAckBin} = recv(Sock),
 
-    DisconnectBin = frame_disconnect(),
+    case SockType of
-    {ok, DisconnectBin} = recv(Sock, 10000),
+        udp ->
-
+            %% another udp client should not affect the first
-    SockType =/= udp andalso
+            %% udp client keepalive check
-        begin
+            timer:sleep(4000),
-            {error, closed} = recv(Sock, 5000)
+            Sock2 = open(SockType),
-        end,
+            ConnBin2 = frame_connect(
-    ok.
+                Client#{clientid => <<"keepalive_test_client2">>},
+                Password
+            ),
+            send(Sock2, ConnBin2),
+            %% first client will be keepalive timeouted in 6s
+            ?assertMatch(
+                {ok, #{
+                    clientid := ClientId1,
+                    reason := {shutdown, {sock_closed, keepalive_timeout}}
+                }},
+                ?block_until(#{?snk_kind := conn_process_terminated}, 8000)
+            );
+        _ ->
+            ?assertMatch(
+                {ok, #{
+                    clientid := ClientId1,
+                    reason := {shutdown, {sock_closed, keepalive_timeout}}
+                }},
+                ?block_until(#{?snk_kind := conn_process_terminated}, 12000)
+            ),
+            Trace = snabbkaffe:collect_trace(),
+            %% conn process should be terminated
+            ?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))),
+            %% socket port should be closed
+            ?assertEqual({error, closed}, recv(Sock, 5000))
+    end,
+    snabbkaffe:stop().
 
 t_hook_connected_disconnected(Cfg) ->
     SockType = proplists:get_value(listener_type, Cfg),
@@ -337,6 +371,8 @@ t_hook_session_subscribed_unsubscribed(Cfg) ->
         error(hook_is_not_running)
     end,
 
+    send(Sock, frame_disconnect()),
+
     close(Sock),
     emqx_hooks:del('session.subscribed', {?MODULE, hook_fun3}),
     emqx_hooks:del('session.unsubscribed', {?MODULE, hook_fun4}).
@@ -373,6 +409,48 @@ t_hook_message_delivered(Cfg) ->
     close(Sock),
     emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}).
 
+t_idle_timeout(Cfg) ->
+    ok = snabbkaffe:start_trace(),
+    SockType = proplists:get_value(listener_type, Cfg),
+    Sock = open(SockType),
+
+    %% need to create udp client by sending something
+    case SockType of
+        udp ->
+            %% nothing to do
+            ok = meck:new(emqx_exproto_gcli, [passthrough, no_history]),
+            ok = meck:expect(
+                emqx_exproto_gcli,
+                async_call,
+                fun(FunName, _Req, _GClient) ->
+                    self() ! {hreply, FunName, ok},
+                    ok
+                end
+            ),
+            %% send request, but nobody can respond to it
+            ClientId = <<"idle_test_client1">>,
+            Client = #{
+                proto_name => <<"demo">>,
+                proto_ver => <<"v0.1">>,
+                clientid => ClientId,
+                keepalive => 5
+            },
+            Password = <<"123456">>,
+            ConnBin = frame_connect(Client, Password),
+            send(Sock, ConnBin),
+            ?assertMatch(
+                {ok, #{reason := {shutdown, idle_timeout}}},
+                ?block_until(#{?snk_kind := conn_process_terminated}, 10000)
+            ),
+            ok = meck:unload(emqx_exproto_gcli);
+        _ ->
+            ?assertMatch(
+                {ok, #{reason := {shutdown, idle_timeout}}},
+                ?block_until(#{?snk_kind := conn_process_terminated}, 10000)
+            )
+    end,
+    snabbkaffe:stop().
+
 %%--------------------------------------------------------------------
 %% Utils
 
@@ -422,6 +500,9 @@ send({ssl, Sock}, Bin) ->
 send({dtls, Sock}, Bin) ->
     ssl:send(Sock, Bin).
 
+recv(Sock) ->
+    recv(Sock, infinity).
+
 recv({tcp, Sock}, Ts) ->
     gen_tcp:recv(Sock, 0, Ts);
 recv({udp, Sock}, Ts) ->

apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf

@@ -2,78 +2,82 @@ emqx_mgmt_api_alarms {
 
     list_alarms_api {
         desc {
-            en: """List alarms"""
+            en: """List currently activated alarms or historical alarms, determined by query parameters."""
-            zh: """列出告警，获取告警列表"""
+            zh: """列出当前激活的告警或历史告警，由查询参数决定。"""
         }
     }
 
     delete_alarms_api {
         desc {
-            en: """Remove all deactivated alarms"""
+            en: """Remove all historical alarms."""
-            zh: """删除所有历史告警（非活跃告警）"""
+            zh: """删除所有历史告警。"""
         }
     }
 
     delete_alarms_api_response204 {
         desc {
-            en: """Remove all deactivated alarms ok"""
+            en: """Historical alarms have been cleared successfully."""
-            zh: """删除所有历史告警（非活跃告警）成功"""
+            zh: """历史告警已成功清除。"""
         }
     }
 
     get_alarms_qs_activated {
         desc {
-            en: """Activate alarms, or deactivate alarms. Default is false"""
+            en: """It is used to specify the alarm type of the query.
-            zh: """活跃中的告警，或历史告警（非活跃告警），默认为 false"""
+When true, it returns the currently activated alarm,
+and when it is false, it returns the historical alarm.
+The default is false."""
+            zh: """用于指定查询的告警类型，
+为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。"""
         }
     }
 
     node {
         desc {
-            en: """Alarm in node"""
+            en: """The name of the node that triggered this alarm."""
-            zh: """告警节点名称"""
+            zh: """触发此告警的节点名称。"""
         }
     }
 
     name {
         desc {
-            en: """Alarm name"""
+            en: """Alarm name, used to distinguish different alarms."""
-            zh: """告警名称"""
+            zh: """告警名称，用于区分不同的告警。"""
         }
     }
 
     message {
         desc {
-            en: """Alarm readable information"""
+            en: """Alarm message, which describes the alarm content in a human-readable format."""
-            zh: """告警信息"""
+            zh: """告警消息，以人类可读的方式描述告警内容。"""
         }
     }
 
     details {
         desc {
-            en: """Alarm details information"""
+            en: """Alarm details, provides more alarm information, mainly for program processing."""
-            zh: """告警详细信息"""
+            zh: """告警详情，提供了更多的告警信息，主要提供给程序处理。"""
         }
     }
 
     duration {
         desc {
-            en: """Alarms duration time; UNIX time stamp, millisecond"""
+            en: """Indicates how long the alarm has lasted, in milliseconds."""
-            zh: """告警持续时间，单位：毫秒"""
+            zh: """表明告警已经持续了多久，单位：毫秒。"""
         }
     }
 
     activate_at {
         desc {
-            en: """Alarms activate time, RFC 3339"""
+            en: """Alarm start time, using rfc3339 standard time format."""
-            zh: """告警开始时间，使用 rfc3339 标准时间格式"""
+            zh: """告警开始时间，使用 rfc3339 标准时间格式。"""
         }
     }
 
     deactivate_at {
         desc {
-            en: """Alarms deactivate time, RFC 3339"""
+            en: """Alarm end time, using rfc3339 standard time format."""
-            zh: """告警结束时间，使用 rfc3339 标准时间格式"""
+            zh: """告警结束时间，使用 rfc3339 标准时间格式。"""
         }
     }
 

apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf

@@ -2,112 +2,97 @@ emqx_mgmt_api_banned {
 
     list_banned_api {
         desc {
-            en: """List banned."""
+            en: """List all currently banned client IDs, usernames and IP addresses."""
-            zh: """列出黑名单"""
+            zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。"""
-        }
-        label {
-            en: """List Banned"""
-            zh: """列出黑名单"""
         }
     }
 
     create_banned_api {
         desc {
-            en: """Create banned."""
+            en: """Add a client ID, username or IP address to the blacklist."""
-            zh: """创建黑名单"""
+            zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。"""
         }
     }
 
     create_banned_api_response400 {
         desc {
-            en: """Banned already existed, or bad args."""
+            en: """Bad request, possibly due to wrong parameters or the existence of a banned object."""
-            zh: """黑名单已存在，或参数格式有错误"""
+            zh: """错误的请求，可能是参数错误或封禁对象已存在等原因。"""
         }
     }
 
     delete_banned_api {
         desc {
-            en: """Delete banned"""
+            en: """Remove a client ID, username or IP address from the blacklist."""
-            zh: """删除黑名单"""
+            zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。"""
         }
     }
 
     delete_banned_api_response404 {
         desc {
-            en: """Banned not found. May be the banned time has been exceeded"""
+            en: """The banned object was not found in the blacklist."""
-            zh: """黑名单未找到，可能为已经超期失效"""
+            zh: """未在黑名单中找到该封禁对象。"""
-        }
-    }
-
-    create_banned {
-        desc {
-            en: """List banned."""
-            zh: """列出黑名单"""
-        }
-        label {
-            en: """List Banned"""
-            zh: """列出黑名单"""
         }
     }
 
     as {
         desc {
-            en: """Banned type clientid, username, peerhost"""
+            en: """Ban method, which can be client ID, username or IP address."""
-            zh: """黑名单类型，可选 clientid、username、peerhost"""
+            zh: """封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。"""
         }
         label {
-            en: """Banned Type"""
+            en: """Ban Method"""
-            zh: """黑名单类型"""
+            zh: """封禁方式"""
         }
     }
     who {
         desc {
-            en: """Client info as banned type"""
+            en: """Ban object, specific client ID, username or IP address."""
-            zh: """设备信息"""
+            zh: """封禁对象，具体的客户端 ID、用户名或者 IP 地址。"""
         }
         label {
-            en: """Banned Info"""
+            en: """Ban Object"""
-            zh: """黑名单信息"""
+            zh: """封禁对象"""
         }
     }
     by {
         desc {
-            en: """Commander"""
+            en: """Initiator of the ban."""
-            zh: """黑名单创建者"""
+            zh: """封禁的发起者。"""
         }
         label {
-            en: """Commander"""
+            en: """Ban Initiator"""
-            zh: """黑名单创建者"""
+            zh: """封禁发起者"""
         }
     }
     reason {
         desc {
-            en: """Banned reason"""
+            en: """Ban reason, record the reason why the current object was banned."""
-            zh: """黑名单创建原因"""
+            zh: """封禁原因，记录当前对象被封禁的原因。"""
         }
         label {
-            en: """Reason"""
+            en: """Ban Reason"""
-            zh: """原因"""
+            zh: """封禁原因"""
         }
     }
     at {
         desc {
-            en: """Create banned time, rfc3339, now if not specified"""
+            en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated."""
-            zh: """黑名单创建时间，默认为当前"""
+            zh: """封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。"""
         }
         label {
-            en: """Create banned time"""
+            en: """Ban Time"""
-            zh: """黑名单创建时间"""
+            zh: """封禁时间"""
         }
     }
     until {
         desc {
-            en: """Cancel banned time, rfc3339, now + 5 minute if not specified"""
+            en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 5 minutes."""
-            zh: """黑名单结束时间，默认为创建时间 + 5 分钟"""
+            zh: """封禁的结束时间，式为 rfc3339，默认为发起操作的时间 + 5 分钟。"""
         }
         label {
-            en: """Cancel banned time"""
+            en: """Ban End Time"""
-            zh: """黑名单结束时间"""
+            zh: """封禁结束时间"""
         }
     }
 }

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},

apps/emqx_management/src/emqx_mgmt_api_app.erl

@@ -22,7 +22,7 @@
 -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
 -export([api_key/2, api_key_by_name/2]).
 -export([validate_name/1]).
--define(TAGS, [<<"API keys">>]).
+-define(TAGS, [<<"API Keys">>]).
 
 namespace() -> "api_key".