Merge remote-tracking branch 'origin/master' into build-with-mix

.github/workflows/build_packages.yaml

@@ -127,7 +127,7 @@ jobs:
       matrix:
         profile: # no EDGE for mac
           - emqx
-          - emqx-ee
+          - emqx-enterprise
         otp:
           - 24.1.5-2
         macos:
@@ -213,7 +213,7 @@ jobs:
         profile: ## all editions for linux
           - emqx-edge
           - emqx
-          - emqx-ee
+          - emqx-enterprise
         otp:
           - 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions
         arch:
@@ -240,9 +240,9 @@ jobs:
         - os: raspbian10
           profile: emqx
         - os: raspbian9
-          profile: emqx-ee
+          profile: emqx-enterprise
         - os: raspbian10
-          profile: emqx-ee
+          profile: emqx-enterprise
 
     defaults:
       run:
@@ -333,7 +333,7 @@ jobs:
         profile: # all editions for docker
           - emqx-edge
           - emqx
-          - emqx-ee
+          - emqx-enterprise
         # NOTE: for docker, only support latest otp version, not a matrix
         otp:
           - 24.1.5-2 # update to latest
@@ -403,7 +403,7 @@ jobs:
         profile:
           - emqx-edge
           - emqx
-          - emqx-ee
+          - emqx-enterprise
         otp:
           - 24.1.5-2
 
@@ -463,7 +463,7 @@ jobs:
     - name: update repo.emqx.io
       if: github.event_name == 'release'
       run: |
-        if [ "${{ matrix. profile }}" = 'emqx-ee' ]; then
+        if [ "${{ matrix.profile }}" = 'emqx-enterprise' ]; then
           BOOL_FLAG_NAME="emqx_ee"
         else
           BOOL_FLAG_NAME="emqx_ce"

.github/workflows/build_slim_packages.yaml

@@ -22,7 +22,7 @@ jobs:
         profile:
         - emqx-edge
         - emqx
-        - emqx-ee
+        - emqx-enterprise
         otp:
         - 24.1.5-2
         os:
@@ -53,7 +53,7 @@ jobs:
       matrix:
         profile:
         - emqx
-        - emqx-ee
+        - emqx-enterprise
         otp:
         - 24.1.5-2
         macos:

.github/workflows/run_api_tests.yaml

@@ -27,7 +27,7 @@ jobs:
       run: |
         echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
         git config --global credential.helper store
-        make emqx-ee-zip
+        make emqx-enterprise-zip
     - uses: actions/upload-artifact@v2
       with:
         name: emqx-broker

.github/workflows/run_fvt_tests.yaml

@@ -38,9 +38,9 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - emqx-edge
           - emqx
-          - emqx-ee
+          - emqx-edge
+          - emqx-enterprise
         cluster_db_backend:
           - mnesia
           - rlog
@@ -88,7 +88,7 @@ jobs:
       matrix:
         profile:
         - emqx
-      # - emqx-ee # TODO test enterprise
+      # - emqx-enterprise # TODO test enterprise
 
     steps:
     - uses: actions/download-artifact@v2

.github/workflows/run_relup_tests.yaml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         profile:
           - emqx
-          - emqx-ee
+          - emqx-enterprise
         otp_vsn:
           - 24.1.5-2
 
@@ -68,7 +68,7 @@ jobs:
         if [ $PROFILE = "emqx" ];then
             broker="emqx-ce"
         else
-            broker="emqx-ee"
+            broker="emqx-enterprise"
         fi
         echo "BROKER=$broker" >> $GITHUB_ENV
 

.github/workflows/run_test_cases.yaml

@@ -23,12 +23,6 @@ jobs:
 
         steps:
         - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
         - name: xref
           run: make xref
         - name: dialyzer
@@ -45,12 +39,6 @@ jobs:
 
         steps:
         - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
         - name: proper
           run: make proper
 

Makefile

@@ -15,8 +15,8 @@ ifeq ($(OS),Windows_NT)
 endif
 
 PROFILE ?= emqx
-REL_PROFILES := emqx emqx-edge emqx-ee
+REL_PROFILES := emqx emqx-edge emqx-enterprise
-PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-ee-pkg
+PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-enterprise-pkg
 PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
 
 CT_NODE_NAME ?= 'test@127.0.0.1'
@@ -182,7 +182,7 @@ ALL_ZIPS = $(REL_PROFILES)
 $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
 
 ## emqx-docker-testing
-## emqx-ee-docker-testing
+## emqx-enterprise-docker-testing
 ## is to directly copy a unzipped zip-package to a
 ## base image such as ubuntu20.04. Mostly for testing
 .PHONY: $(REL_PROFILES:%=%-docker-testing)

apps/emqx/etc/emqx.conf

@@ -577,8 +577,9 @@ mqtt {
   ## @doc mqtt.max_topic_levels
   ## ValueType: Integer
   ## Range: [1, 65535]
-  ## Default: 65535
+  ## Default: 128
-  max_topic_levels = 65535
+  ## Depth so big may lead to subscribing performance issues
+  max_topic_levels = 128
 
   ## Maximum QoS allowed.
   ##

apps/emqx/src/emqx_persistent_session.erl

@@ -18,6 +18,7 @@
 
 -export([ is_store_enabled/0
         , init_db_backend/0
+        , storage_type/0
         ]).
 
 -export([ discard/2
@@ -82,9 +83,17 @@
 init_db_backend() ->
     case is_store_enabled() of
         true  ->
-            ok = emqx_trie:create_session_trie(),
+            StorageType = storage_type(),
-            emqx_persistent_session_mnesia_backend:create_tables(),
+            ok = emqx_trie:create_session_trie(StorageType),
-            persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_backend),
+            ok = emqx_session_router:create_router_tab(StorageType),
+            case StorageType of
+                disc ->
+                    emqx_persistent_session_mnesia_disc_backend:create_tables(),
+                    persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_disc_backend);
+                ram ->
+                    emqx_persistent_session_mnesia_ram_backend:create_tables(),
+                    persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_ram_backend)
+            end,
             ok;
         false ->
             persistent_term:put(?db_backend_key, emqx_persistent_session_dummy_backend),
@@ -94,6 +103,9 @@ init_db_backend() ->
 is_store_enabled() ->
     emqx_config:get(?is_enabled_key).
 
+storage_type() ->
+    emqx_config:get(?storage_type_key).
+
 %%--------------------------------------------------------------------
 %% Session message ADT API
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_persistent_session.hrl

@@ -14,9 +14,14 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--define(SESSION_STORE, emqx_session_store).
+-define(SESSION_STORE_DISC, emqx_session_store_disc).
--define(SESS_MSG_TAB, emqx_session_msg).
+-define(SESSION_STORE_RAM, emqx_session_store_ram).
--define(MSG_TAB, emqx_persistent_msg).
+
+-define(SESS_MSG_TAB_DISC, emqx_session_msg_disc).
+-define(SESS_MSG_TAB_RAM, emqx_session_msg_ram).
+
+-define(MSG_TAB_DISC, emqx_persistent_msg_disc).
+-define(MSG_TAB_RAM, emqx_persistent_msg_ram).
 
 -record(session_store, { client_id        :: binary()
                        , expiry_interval  :: non_neg_integer()
@@ -28,6 +33,7 @@
 
 -define(db_backend_key, [persistent_session_store, db_backend]).
 -define(is_enabled_key, [persistent_session_store, enabled]).
+-define(storage_type_key, [persistent_session_store, storage_type]).
 -define(msg_retain, [persistent_session_store, max_retain_undelivered]).
 
 -define(db_backend, (persistent_term:get(?db_backend_key))).

apps/emqx/src/emqx_persistent_session_mnesia_disc_backend.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_persistent_session_mnesia_backend).
+-module(emqx_persistent_session_mnesia_disc_backend).
 
 -include("emqx.hrl").
 -include("emqx_persistent_session.hrl").
@@ -36,7 +36,7 @@
         ]).
 
 create_tables() ->
-    ok = mria:create_table(?SESSION_STORE, [
+    ok = mria:create_table(?SESSION_STORE_DISC, [
                 {type, set},
                 {rlog_shard, ?PERSISTENT_SESSION_SHARD},
                 {storage, disc_copies},
@@ -44,7 +44,7 @@ create_tables() ->
                 {attributes, record_info(fields, session_store)},
                 {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
 
-    ok = mria:create_table(?SESS_MSG_TAB, [
+    ok = mria:create_table(?SESS_MSG_TAB_DISC, [
                 {type, ordered_set},
                 {rlog_shard, ?PERSISTENT_SESSION_SHARD},
                 {storage, disc_copies},
@@ -53,7 +53,7 @@ create_tables() ->
                 {storage_properties, [{ets, [{read_concurrency, true},
                                              {write_concurrency, true}]}]}]),
 
-    ok = mria:create_table(?MSG_TAB, [
+    ok = mria:create_table(?MSG_TAB_DISC, [
                 {type, ordered_set},
                 {rlog_shard, ?PERSISTENT_SESSION_SHARD},
                 {storage, disc_copies},
@@ -63,43 +63,43 @@ create_tables() ->
                                              {write_concurrency, true}]}]}]).
 
 first_session_message() ->
-    mnesia:dirty_first(?SESS_MSG_TAB).
+    mnesia:dirty_first(?SESS_MSG_TAB_DISC).
 
 next_session_message(Key) ->
-    mnesia:dirty_next(?SESS_MSG_TAB, Key).
+    mnesia:dirty_next(?SESS_MSG_TAB_DISC, Key).
 
 first_message_id() ->
-    mnesia:dirty_first(?MSG_TAB).
+    mnesia:dirty_first(?MSG_TAB_DISC).
 
 next_message_id(Key) ->
-    mnesia:dirty_next(?MSG_TAB, Key).
+    mnesia:dirty_next(?MSG_TAB_DISC, Key).
 
 delete_message(Key) ->
-    mria:dirty_delete(?MSG_TAB, Key).
+    mria:dirty_delete(?MSG_TAB_DISC, Key).
 
 delete_session_message(Key) ->
-    mria:dirty_delete(?SESS_MSG_TAB, Key).
+    mria:dirty_delete(?SESS_MSG_TAB_DISC, Key).
 
 put_session_store(SS) ->
-    mria:dirty_write(?SESSION_STORE, SS).
+    mria:dirty_write(?SESSION_STORE_DISC, SS).
 
 delete_session_store(ClientID) ->
-    mria:dirty_delete(?SESSION_STORE, ClientID).
+    mria:dirty_delete(?SESSION_STORE_DISC, ClientID).
 
 lookup_session_store(ClientID) ->
-    case mnesia:dirty_read(?SESSION_STORE, ClientID) of
+    case mnesia:dirty_read(?SESSION_STORE_DISC, ClientID) of
         [] -> none;
         [SS] -> {value, SS}
     end.
 
 put_session_message(SessMsg) ->
-    mria:dirty_write(?SESS_MSG_TAB, SessMsg).
+    mria:dirty_write(?SESS_MSG_TAB_DISC, SessMsg).
 
 put_message(Msg) ->
-    mria:dirty_write(?MSG_TAB, Msg).
+    mria:dirty_write(?MSG_TAB_DISC, Msg).
 
 get_message(MsgId) ->
-    case mnesia:read(?MSG_TAB, MsgId) of
+    case mnesia:read(?MSG_TAB_DISC, MsgId) of
         [] -> error({msg_not_found, MsgId});
         [Msg] -> Msg
     end.

apps/emqx/src/emqx_persistent_session_mnesia_ram_backend.erl

@@ -0,0 +1,110 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_persistent_session_mnesia_ram_backend).
+
+-include("emqx.hrl").
+-include("emqx_persistent_session.hrl").
+
+-export([ create_tables/0
+        , first_message_id/0
+        , next_message_id/1
+        , delete_message/1
+        , first_session_message/0
+        , next_session_message/1
+        , delete_session_message/1
+        , put_session_store/1
+        , delete_session_store/1
+        , lookup_session_store/1
+        , put_session_message/1
+        , put_message/1
+        , get_message/1
+        , ro_transaction/1
+        ]).
+
+create_tables() ->
+    ok = mria:create_table(?SESSION_STORE_RAM, [
+                {type, set},
+                {rlog_shard, ?PERSISTENT_SESSION_SHARD},
+                {storage, ram_copies},
+                {record_name, session_store},
+                {attributes, record_info(fields, session_store)},
+                {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+
+    ok = mria:create_table(?SESS_MSG_TAB_RAM, [
+                {type, ordered_set},
+                {rlog_shard, ?PERSISTENT_SESSION_SHARD},
+                {storage, ram_copies},
+                {record_name, session_msg},
+                {attributes, record_info(fields, session_msg)},
+                {storage_properties, [{ets, [{read_concurrency, true},
+                                             {write_concurrency, true}]}]}]),
+
+    ok = mria:create_table(?MSG_TAB_RAM, [
+                {type, ordered_set},
+                {rlog_shard, ?PERSISTENT_SESSION_SHARD},
+                {storage, ram_copies},
+                {record_name, message},
+                {attributes, record_info(fields, message)},
+                {storage_properties, [{ets, [{read_concurrency, true},
+                                             {write_concurrency, true}]}]}]).
+
+first_session_message() ->
+    mnesia:dirty_first(?SESS_MSG_TAB_RAM).
+
+next_session_message(Key) ->
+    mnesia:dirty_next(?SESS_MSG_TAB_RAM, Key).
+
+first_message_id() ->
+    mnesia:dirty_first(?MSG_TAB_RAM).
+
+next_message_id(Key) ->
+    mnesia:dirty_next(?MSG_TAB_RAM, Key).
+
+delete_message(Key) ->
+    mria:dirty_delete(?MSG_TAB_RAM, Key).
+
+delete_session_message(Key) ->
+    mria:dirty_delete(?SESS_MSG_TAB_RAM, Key).
+
+put_session_store(SS) ->
+    mria:dirty_write(?SESSION_STORE_RAM, SS).
+
+delete_session_store(ClientID) ->
+    mria:dirty_delete(?SESSION_STORE_RAM, ClientID).
+
+lookup_session_store(ClientID) ->
+    case mnesia:dirty_read(?SESSION_STORE_RAM, ClientID) of
+        [] -> none;
+        [SS] -> {value, SS}
+    end.
+
+put_session_message(SessMsg) ->
+    mria:dirty_write(?SESS_MSG_TAB_RAM, SessMsg).
+
+put_message(Msg) ->
+    mria:dirty_write(?MSG_TAB_RAM, Msg).
+
+get_message(MsgId) ->
+    case mnesia:read(?MSG_TAB_RAM, MsgId) of
+        [] -> error({msg_not_found, MsgId});
+        [Msg] -> Msg
+    end.
+
+ro_transaction(Fun) ->
+    {atomic, Res} = mria:ro_transaction(?PERSISTENT_SESSION_SHARD, Fun),
+    Res.
+

apps/emqx/src/emqx_router_utils.erl

@@ -20,8 +20,10 @@
 
 -export([ delete_direct_route/2
         , delete_trie_route/2
+        , delete_session_trie_route/2
         , insert_direct_route/2
         , insert_trie_route/2
+        , insert_session_trie_route/2
         , maybe_trans/3
         ]).
 
@@ -30,8 +32,14 @@ insert_direct_route(Tab, Route) ->
 
 insert_trie_route(RouteTab, Route = #route{topic = Topic}) ->
     case mnesia:wread({RouteTab, Topic}) of
-        [] when RouteTab =:= emqx_route         -> emqx_trie:insert(Topic);
+        [] -> emqx_trie:insert(Topic);
-        [] when RouteTab =:= emqx_session_route -> emqx_trie:insert_session(Topic);
+        _  -> ok
+    end,
+    mnesia:write(RouteTab, Route, sticky_write).
+
+insert_session_trie_route(RouteTab, Route = #route{topic = Topic}) ->
+    case mnesia:wread({RouteTab, Topic}) of
+        []  -> emqx_trie:insert_session(Topic);
         _  -> ok
     end,
     mnesia:write(RouteTab, Route, sticky_write).
@@ -39,14 +47,20 @@ insert_trie_route(RouteTab, Route = #route{topic = Topic}) ->
 delete_direct_route(RouteTab, Route) ->
     mria:dirty_delete_object(RouteTab, Route).
 
-delete_trie_route(RouteTab, Route = #route{topic = Topic}) ->
+delete_trie_route(RouteTab, Route) ->
+    delete_trie_route(RouteTab, Route, normal).
+
+delete_session_trie_route(RouteTab, Route) ->
+    delete_trie_route(RouteTab, Route, session).
+
+delete_trie_route(RouteTab, Route = #route{topic = Topic}, Type) ->
     case mnesia:wread({RouteTab, Topic}) of
         [R] when R =:= Route ->
             %% Remove route and trie
             ok = mnesia:delete_object(RouteTab, Route, sticky_write),
-            case RouteTab of
+            case Type of
-                emqx_route         -> emqx_trie:delete(Topic);
+                normal  -> emqx_trie:delete(Topic);
-                emqx_session_route -> emqx_trie:delete_session(Topic)
+                session -> emqx_trie:delete_session(Topic)
             end;
         [_|_]   ->
             %% Remove route only

apps/emqx/src/emqx_schema.erl

@@ -161,19 +161,50 @@ roots(low) ->
 fields("persistent_session_store") ->
     [ {"enabled",
        sc(boolean(),
-          #{ default => "false"
+          #{ default => false
+           , description => """
+Use the database to store information about persistent sessions.
+This makes it possible to migrate a client connection to another
+cluster node if a node is stopped.
+"""
+           })},
+      {"storage_type",
+       sc(hoconsc:union([ram, disc]),
+          #{ default => disc
+           , description => """
+Store information about persistent sessions on disc or in ram.
+If ram is chosen, all information about persistent sessions remains
+as long as at least one node in a cluster is alive to keep the information.
+If disc is chosen, the information is persisted on disc and will survive
+cluster restart, at the price of more disc usage and less throughput.
+"""
            })},
       {"max_retain_undelivered",
        sc(duration(),
           #{ default => "1h"
+           , description => """
+The time messages that was not delivered to a persistent session
+is stored before being garbage collected if the node the previous
+session was handled on restarts of is stopped.
+"""
            })},
       {"message_gc_interval",
        sc(duration(),
           #{ default => "1h"
+           , description => """
+The starting interval for garbage collection of undelivered messages to
+a persistent session. This affects how often the \"max_retain_undelivered\"
+is checked for removal.
+"""
            })},
       {"session_message_gc_interval",
        sc(duration(),
           #{ default => "1m"
+           , description => """
+The starting interval for garbage collection of transient data for
+persistent session messages. This does not affect the life time length
+of persistent session messages.
+"""
            })}
     ];
 

apps/emqx/src/emqx_session_router.erl

@@ -24,12 +24,8 @@
 
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
-%% Mnesia bootstrap
--export([mnesia/1]).
-
--boot_mnesia({mnesia, [boot]}).
-
 -export([ create_init_tab/0
+        , create_router_tab/1
         , start_link/2]).
 
 %% Route APIs
@@ -60,7 +56,8 @@
 
 -type(dest() :: node() | {group(), node()}).
 
--define(ROUTE_TAB, emqx_session_route).
+-define(ROUTE_RAM_TAB, emqx_session_route_ram).
+-define(ROUTE_DISC_TAB, emqx_session_route_disc).
 
 -define(SESSION_INIT_TAB, session_init_tab).
 
@@ -68,13 +65,22 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_router_tab(disc) ->
-    ok = mria:create_table(?ROUTE_TAB, [
+    ok = mria:create_table(?ROUTE_DISC_TAB, [
                 {type, bag},
                 {rlog_shard, ?ROUTE_SHARD},
                 {storage, disc_copies},
                 {record_name, route},
                 {attributes, record_info(fields, route)},
+                {storage_properties, [{ets, [{read_concurrency, true},
+                                             {write_concurrency, true}]}]}]);
+create_router_tab(ram) ->
+    ok = mria:create_table(?ROUTE_RAM_TAB, [
+                {type, bag},
+                {rlog_shard, ?ROUTE_SHARD},
+                {storage, ram_copies},
+                {record_name, route},
+                {attributes, record_info(fields, route)},
                 {storage_properties, [{ets, [{read_concurrency, true},
                                              {write_concurrency, true}]}]}]).
 
@@ -103,11 +109,11 @@ do_add_route(Topic, SessionID) when is_binary(Topic) ->
         false ->
             case emqx_topic:wildcard(Topic) of
                 true  ->
-                    Fun = fun emqx_router_utils:insert_trie_route/2,
+                    Fun = fun emqx_router_utils:insert_session_trie_route/2,
-                    emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route],
+                    emqx_router_utils:maybe_trans(Fun, [route_tab(), Route],
                                                   ?PERSISTENT_SESSION_SHARD);
                 false ->
-                    emqx_router_utils:insert_direct_route(?ROUTE_TAB, Route)
+                    emqx_router_utils:insert_direct_route(route_tab(), Route)
             end
     end.
 
@@ -136,10 +142,10 @@ do_delete_route(Topic, SessionID) ->
     Route = #route{topic = Topic, dest = SessionID},
     case emqx_topic:wildcard(Topic) of
         true  ->
-            Fun = fun emqx_router_utils:delete_trie_route/2,
+            Fun = fun emqx_router_utils:delete_session_trie_route/2,
-            emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route], ?PERSISTENT_SESSION_SHARD);
+            emqx_router_utils:maybe_trans(Fun, [route_tab(), Route], ?PERSISTENT_SESSION_SHARD);
         false ->
-            emqx_router_utils:delete_direct_route(?ROUTE_TAB, Route)
+            emqx_router_utils:delete_direct_route(route_tab(), Route)
     end.
 
 %% @doc Print routes to a topic
@@ -273,4 +279,10 @@ init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) ->
 %%--------------------------------------------------------------------
 
 lookup_routes(Topic) ->
-    ets:lookup(?ROUTE_TAB, Topic).
+    ets:lookup(route_tab(), Topic).
+
+route_tab() ->
+    case emqx_persistent_session:storage_type() of
+        disc -> ?ROUTE_DISC_TAB;
+        ram  -> ?ROUTE_RAM_TAB
+    end.

apps/emqx/src/emqx_sys_mon.erl

@@ -187,11 +187,20 @@ suppress(Key, SuccFun, State = #{events := Events}) ->
 
 procinfo(Pid) ->
     [{pid, Pid} | procinfo_l(emqx_vm:get_process_gc_info(Pid))] ++
+    get_proc_lib_initial_call(Pid) ++
     procinfo_l(emqx_vm:get_process_info(Pid)).
 
 procinfo_l(undefined) -> [];
 procinfo_l(List) -> List.
 
+get_proc_lib_initial_call(Pid) ->
+    case proc_lib:initial_call(Pid) of
+        false ->
+            [];
+        InitialCall ->
+            [{proc_lib_initial_call, InitialCall}]
+    end.
+
 portinfo(Port) ->
     [{port, Port} | erlang:port_info(Port)].
 

apps/emqx/src/emqx_trie.erl

@@ -20,7 +20,7 @@
 
 %% Mnesia bootstrap
 -export([ mnesia/1
-        , create_session_trie/0
+        , create_session_trie/1
         ]).
 
 -boot_mnesia({mnesia, [boot]}).
@@ -48,7 +48,8 @@
 -endif.
 
 -define(TRIE, emqx_trie).
--define(SESSION_TRIE, emqx_session_trie).
+-define(SESSION_DISC_TRIE, emqx_session_trie_disc).
+-define(SESSION_RAM_TRIE, emqx_session_trie_ram).
 -define(PREFIX(Prefix), {Prefix, 0}).
 -define(TOPIC(Topic), {Topic, 1}).
 
@@ -75,16 +76,27 @@ mnesia(boot) ->
                 {type, ordered_set},
                 {storage_properties, StoreProps}]).
 
-create_session_trie() ->
+create_session_trie(disc) ->
     StoreProps = [{ets, [{read_concurrency, true},
                          {write_concurrency, true}
                         ]}],
-    ok = mria:create_table(?SESSION_TRIE,
+    ok = mria:create_table(?SESSION_DISC_TRIE,
                            [{rlog_shard, ?ROUTE_SHARD},
                             {storage, disc_copies},
                             {record_name, ?TRIE},
                             {attributes, record_info(fields, ?TRIE)},
                             {type, ordered_set},
+                            {storage_properties, StoreProps}]);
+create_session_trie(ram) ->
+    StoreProps = [{ets, [{read_concurrency, true},
+                         {write_concurrency, true}
+                        ]}],
+    ok = mria:create_table(?SESSION_RAM_TRIE,
+                           [{rlog_shard, ?ROUTE_SHARD},
+                            {storage, ram_copies},
+                            {record_name, ?TRIE},
+                            {attributes, record_info(fields, ?TRIE)},
+                            {type, ordered_set},
                             {storage_properties, StoreProps}]).
 
 %%--------------------------------------------------------------------
@@ -98,7 +110,7 @@ insert(Topic) when is_binary(Topic) ->
 
 -spec(insert_session(emqx_topic:topic()) -> ok).
 insert_session(Topic) when is_binary(Topic) ->
-    insert(Topic, ?SESSION_TRIE).
+    insert(Topic, session_trie()).
 
 insert(Topic, Trie) when is_binary(Topic) ->
     {TopicKey, PrefixKeys} = make_keys(Topic),
@@ -115,7 +127,7 @@ delete(Topic) when is_binary(Topic) ->
 %% @doc Delete a topic filter from the trie.
 -spec(delete_session(emqx_topic:topic()) -> ok).
 delete_session(Topic) when is_binary(Topic) ->
-    delete(Topic, ?SESSION_TRIE).
+    delete(Topic, session_trie()).
 
 delete(Topic, Trie) when is_binary(Topic) ->
     {TopicKey, PrefixKeys} = make_keys(Topic),
@@ -131,7 +143,7 @@ match(Topic) when is_binary(Topic) ->
 
 -spec(match_session(emqx_topic:topic()) -> list(emqx_topic:topic())).
 match_session(Topic) when is_binary(Topic) ->
-    match(Topic, ?SESSION_TRIE).
+    match(Topic, session_trie()).
 
 match(Topic, Trie) when is_binary(Topic) ->
     Words = emqx_topic:words(Topic),
@@ -154,7 +166,7 @@ match(Topic, Trie) when is_binary(Topic) ->
 empty() -> empty(?TRIE).
 
 empty_session() ->
-    empty(?SESSION_TRIE).
+    empty(session_trie()).
 
 empty(Trie) -> ets:first(Trie) =:= '$end_of_table'.
 
@@ -164,12 +176,19 @@ lock_tables() ->
 
 -spec lock_session_tables() -> ok.
 lock_session_tables() ->
-    mnesia:write_lock_table(?SESSION_TRIE).
+    mnesia:write_lock_table(session_trie()).
 
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
 
+session_trie() ->
+    case emqx_persistent_session:storage_type() of
+        disc -> ?SESSION_DISC_TRIE;
+        ram  -> ?SESSION_RAM_TRIE
+    end.
+
+
 make_keys(Topic) ->
     Words = emqx_topic:words(Topic),
     {?TOPIC(Topic), [?PREFIX(Prefix) || Prefix <- make_prefixes(Words)]}.

apps/emqx/src/emqx_vm.erl

@@ -61,7 +61,7 @@
                          ]).
 
 -define(PROCESS_INFO_KEYS, [initial_call,
-                            current_function,
+                            current_stacktrace,
                             registered_name,
                             status,
                             message_queue_len,

apps/emqx/test/emqx_channel_SUITE.erl

@@ -59,7 +59,7 @@ mqtt_conf() ->
       max_inflight => 32,max_mqueue_len => 1000,
       max_packet_size => 1048576,max_qos_allowed => 2,
       max_subscriptions => infinity,max_topic_alias => 65535,
-      max_topic_levels => 65535,mqueue_default_priority => lowest,
+      max_topic_levels => 128,mqueue_default_priority => lowest,
       mqueue_priorities => disabled,mqueue_store_qos0 => true,
       peer_cert_as_clientid => disabled,
       peer_cert_as_username => disabled,
@@ -200,7 +200,7 @@ t_chan_caps(_) ->
      #{max_clientid_len := 65535,
        max_qos_allowed := 2,
        max_topic_alias := 65535,
-       max_topic_levels := 65535,
+       max_topic_levels := 128,
        retain_available := true,
        shared_subscription := true,
        subscription_identifiers := true,

apps/emqx/test/emqx_persistent_session_SUITE.erl

@@ -50,13 +50,19 @@ groups() ->
     SnabbkaffeTCs = [TC || TC <- TCs, is_snabbkaffe_tc(TC)],
     GCTests  = [TC || TC <- TCs, is_gc_tc(TC)],
     OtherTCs = (TCs -- SnabbkaffeTCs) -- GCTests,
-    [ {persistent_store_enabled, [ {group, no_kill_connection_process}
+    [ {persistent_store_enabled, [ {group, ram_tables}
-                                 , {group,    kill_connection_process}
+                                 , {group, disc_tables}
-                                 , {group, snabbkaffe}
-                                 , {group, gc_tests}
                                  ]}
     , {persistent_store_disabled, [ {group, no_kill_connection_process}
                                   ]}
+    , {  ram_tables, [], [ {group, no_kill_connection_process}
+                         , {group,    kill_connection_process}
+                         , {group, snabbkaffe}
+                         , {group, gc_tests}]}
+    , { disc_tables, [], [ {group, no_kill_connection_process}
+                         , {group,    kill_connection_process}
+                         , {group, snabbkaffe}
+                         , {group, gc_tests}]}
     , {no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]}
     , {   kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]}
     , {snabbkaffe, [], [{group, tcp_snabbkaffe}, {group, quic_snabbkaffe}, {group, ws_snabbkaffe}]}
@@ -76,15 +82,23 @@ is_gc_tc(TC) ->
     re:run(atom_to_list(TC), "^t_gc_") /= nomatch.
 
 init_per_group(persistent_store_enabled, Config) ->
+    [{persistent_store_enabled, true}|Config];
+init_per_group(Group, Config) when Group =:= ram_tables; Group =:= disc_tables ->
     %% Start Apps
+    Reply = case Group =:= ram_tables of
+                true  -> ram;
+                false -> disc
+            end,
     emqx_common_test_helpers:boot_modules(all),
     meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx_config, get, fun(?is_enabled_key) -> true;
+    meck:expect(emqx_config, get, fun(?storage_type_key) -> Reply;
-                                   (Other) -> meck:passthrough([Other])
+                                     (?is_enabled_key) -> true;
+                                     (Other) -> meck:passthrough([Other])
                                 end),
     emqx_common_test_helpers:start_apps([], fun set_special_confs/1),
     ?assertEqual(true, emqx_persistent_session:is_store_enabled()),
-    [{persistent_store_enabled, true}|Config];
+    ?assertEqual(Reply, emqx_persistent_session:storage_type()),
+    Config;
 init_per_group(persistent_store_disabled, Config) ->
     %% Start Apps
     emqx_common_test_helpers:boot_modules(all),
@@ -125,15 +139,20 @@ init_per_group(gc_tests, Config) ->
                         receive stop -> ok end
                 end),
     meck:new(mnesia, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(mnesia, dirty_first, fun(?SESS_MSG_TAB) -> ets:first(SessionMsgEts);
+    meck:expect(mnesia, dirty_first, fun(?SESS_MSG_TAB_RAM) -> ets:first(SessionMsgEts);
-                                        (?MSG_TAB) -> ets:first(MsgEts);
+                                        (?SESS_MSG_TAB_DISC) -> ets:first(SessionMsgEts);
-                                        (X) -> meck:passthrough(X)
+                                        (?MSG_TAB_RAM) -> ets:first(MsgEts);
+                                        (?MSG_TAB_DISC) -> ets:first(MsgEts);
+                                        (X) -> meck:passthrough([X])
                                      end),
-    meck:expect(mnesia, dirty_next, fun(?SESS_MSG_TAB, X) -> ets:next(SessionMsgEts, X);
+    meck:expect(mnesia, dirty_next, fun(?SESS_MSG_TAB_RAM, X) -> ets:next(SessionMsgEts, X);
-                                       (?MSG_TAB, X) -> ets:next(MsgEts, X);
+                                       (?SESS_MSG_TAB_DISC, X) -> ets:next(SessionMsgEts, X);
+                                       (?MSG_TAB_RAM, X) -> ets:next(MsgEts, X);
+                                       (?MSG_TAB_DISC, X) -> ets:next(MsgEts, X);
                                        (Tab, X) -> meck:passthrough([Tab, X])
                                     end),
-    meck:expect(mnesia, dirty_delete, fun(?MSG_TAB, X) -> ets:delete(MsgEts, X);
+    meck:expect(mnesia, dirty_delete, fun(?MSG_TAB_RAM, X) -> ets:delete(MsgEts, X);
+                                         (?MSG_TAB_DISC, X) -> ets:delete(MsgEts, X);
                                          (Tab, X) -> meck:passthrough([Tab, X])
                                       end),
     [{store_owner, Pid}, {session_msg_store, SessionMsgEts}, {msg_store, MsgEts} | Config].
@@ -154,7 +173,7 @@ end_per_group(gc_tests, Config) ->
     meck:unload(mnesia),
     ?config(store_owner, Config) ! stop,
     ok;
-end_per_group(persistent_store_enabled, _Config) ->
+end_per_group(Group, _Config) when Group =:= ram_tables; Group =:= disc_tables ->
     meck:unload(emqx_config),
     emqx_common_test_helpers:stop_apps([]);
 end_per_group(persistent_store_disabled, _Config) ->
@@ -250,7 +269,7 @@ maybe_kill_connection_process(ClientId, Config) ->
     end.
 
 wait_for_cm_unregister(ClientId) ->
-    wait_for_cm_unregister(ClientId, 10).
+    wait_for_cm_unregister(ClientId, 100).
 
 wait_for_cm_unregister(_ClientId, 0) ->
     error(cm_did_not_unregister);
@@ -884,12 +903,16 @@ t_snabbkaffe_buffered_messages(Config) ->
            %% Make the resume init phase wait until the first message is delivered.
            ?force_ordering( #{ ?snk_kind := ps_worker_deliver },
                             #{ ?snk_kind := ps_resume_end }),
+           Parent = self(),
            spawn_link(fun() ->
                               ?block_until(#{?snk_kind := ps_marker_pendings_msgs}, infinity, 5000),
-                              publish(Topic, Payloads2, true)
+                              publish(Topic, Payloads2, true),
+                              Parent ! publish_done,
+                              ok
                       end),
            {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]),
            {ok, _} = emqtt:ConnFun(Client2),
+           receive publish_done -> ok after 10000 -> error(too_long_to_publish) end,
            Msgs = receive_messages(length(Payloads1) + length(Payloads2) + 1),
            ReceivedPayloads = [P || #{ payload := P } <- Msgs],
            ?assertEqual(lists:sort(Payloads1 ++ Payloads2),

apps/emqx/test/emqx_sys_mon_SUITE.erl

@@ -71,23 +71,48 @@ init_per_testcase(t_sys_mon2, Config) ->
            (_) -> ok
         end),
     Config;
+init_per_testcase(t_procinfo, Config) ->
+    emqx_common_test_helpers:boot_modules(all),
+    emqx_common_test_helpers:start_apps([]),
+    ok = meck:new(emqx_vm, [passthrough, no_history]),
+    Config;
 init_per_testcase(_, Config) ->
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.
 
+end_per_testcase(t_procinfo, _Config) ->
+    ok = meck:unload(emqx_vm),
+    emqx_common_test_helpers:stop_apps([]);
 end_per_testcase(_, _Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
 t_procinfo(_) ->
-    ok = meck:new(emqx_vm, [passthrough, no_history]),
-    ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end),
-    ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> [] end),
-    ?assertEqual([{pid, undefined}], emqx_sys_mon:procinfo(undefined)),
     ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end),
     ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> undefined end),
-    ?assertEqual([{pid, self()}], emqx_sys_mon:procinfo(self())),
+    ?assertEqual([{pid, self()}], emqx_sys_mon:procinfo(self())).
-    ok = meck:unload(emqx_vm).
+
+t_procinfo_initial_call_and_stacktrace(_) ->
+    SomePid = proc_lib:spawn(?MODULE, some_function, [self(), arg2]),
+    receive
+        {spawned, SomePid} ->
+            ok
+    after 100 ->
+            error(process_not_spawned)
+    end,
+    ProcInfo = emqx_sys_mon:procinfo(SomePid),
+    ?assertEqual(
+       {?MODULE, some_function, ['Argument__1','Argument__2']},
+       proplists:get_value(proc_lib_initial_call, ProcInfo)),
+    ?assertMatch(
+       [{?MODULE, some_function, 2,
+         [{file, _},
+          {line, _}]},
+        {proc_lib, init_p_do_apply, 3,
+         [{file, _},
+          {line, _}]}],
+       proplists:get_value(current_stacktrace, ProcInfo)),
+    SomePid ! stop.
 
 t_sys_mon(_Config) ->
     lists:foreach(
@@ -119,3 +144,10 @@ validate_sys_mon_info(PidOrPort, SysMonName, ValidateInfo, InfoOrPort) ->
     emqtt:stop(C).
 
 fmt(Fmt, Args) -> lists:flatten(io_lib:format(Fmt, Args)).
+
+some_function(Parent, _Arg2) ->
+    Parent ! {spawned, self()},
+    receive
+        stop ->
+            ok
+    end.

apps/emqx/test/emqx_trace_SUITE.erl

@@ -22,7 +22,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
-
+-import(emqx_trace_handler_SUITE, [filesync/2]).
 -record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}).
 
 %%--------------------------------------------------------------------
@@ -237,15 +237,16 @@ t_client_event(_Config) ->
     emqtt:ping(Client),
     ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]),
     ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]),
-    ct:sleep(200),
     ok = emqx_trace:create([{<<"name">>, <<"test_topic">>},
         {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]),
-    ct:sleep(200),
+    ok = filesync(Name, clientid),
+    ok = filesync(<<"test_topic">>, topic),
     {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)),
     ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]),
     ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]),
     ok = emqtt:disconnect(Client),
-    ct:sleep(200),
+    ok = filesync(Name, clientid),
+    ok = filesync(<<"test_topic">>, topic),
     {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)),
     {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)),
     ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]),
@@ -301,7 +302,7 @@ t_download_log(_Config) ->
     {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
     {ok, _} = emqtt:connect(Client),
     [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)],
-    ct:sleep(100),
+    ok = filesync(Name, clientid),
     {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []),
     ?assert(filelib:file_size(ZipFile) > 0),
     ok = emqtt:disconnect(Client),

apps/emqx/test/emqx_trace_handler_SUITE.erl

@@ -62,7 +62,9 @@ t_trace_clientid(_Config) ->
         emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"),
     {error, {handler_not_added, {file_error, ".", eisdir}}} =
         emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
-    ct:sleep(100),
+    ok = filesync(<<"client">>, clientid),
+    ok = filesync(<<"client2">>, clientid),
+    ok = filesync(<<"client3">>, clientid),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/client.log")),
@@ -83,7 +85,10 @@ t_trace_clientid(_Config) ->
     emqtt:connect(T),
     emqtt:publish(T, <<"a/b/c">>, <<"hi">>),
     emqtt:ping(T),
-    ct:sleep(200),
+
+    ok = filesync(<<"client">>, clientid),
+    ok = filesync(<<"client2">>, clientid),
+    ok = filesync(<<"client3">>, clientid),
 
     %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log".
     {ok, Bin} = file:read_file("tmp/client.log"),
@@ -109,7 +114,8 @@ t_trace_topic(_Config) ->
     emqx_logger:set_log_level(debug),
     ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
     ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
-    ct:sleep(100),
+    ok = filesync(<<"x/#">>, topic),
+    ok = filesync(<<"y/#">>, topic),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/topic_trace_x.log")),
@@ -128,7 +134,8 @@ t_trace_topic(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ct:sleep(200),
+    ok = filesync(<<"x/#">>, topic),
+    ok = filesync(<<"y/#">>, topic),
 
     {ok, Bin} = file:read_file("tmp/topic_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -152,8 +159,8 @@ t_trace_ip_address(_Config) ->
     %% Start tracing
     ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
     ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"),
-    ct:sleep(100),
+    ok = filesync(<<"127.0.0.1">>, ip_address),
-
+    ok = filesync(<<"192.168.1.1">>, ip_address),
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/ip_trace_x.log")),
     ?assert(filelib:is_regular("tmp/ip_trace_y.log")),
@@ -173,7 +180,8 @@ t_trace_ip_address(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ct:sleep(200),
+    ok = filesync(<<"127.0.0.1">>, ip_address),
+    ok = filesync(<<"192.168.1.1">>, ip_address),
 
     {ok, Bin} = file:read_file("tmp/ip_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -189,3 +197,19 @@ t_trace_ip_address(_Config) ->
     {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>),
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()).
+
+filesync(Name, Type) ->
+    filesync(Name, Type, 3).
+
+%% sometime the handler process is not started yet.
+filesync(_Name, _Type, 0) -> ok;
+filesync(Name, Type, Retry) ->
+    try
+        Handler = binary_to_atom(<<"trace_",
+            (atom_to_binary(Type))/binary, "_", Name/binary>>),
+        ok = logger_disk_log_h:filesync(Handler)
+    catch E:R ->
+        ct:pal("Filesync error:~p ~p~n", [{Name, Type, Retry}, {E, R}]),
+        ct:sleep(100),
+        filesync(Name, Type, Retry - 1)
+    end.

apps/emqx_authn/src/emqx_authn_api.erl

@@ -21,6 +21,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include("emqx_authn.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -import(hoconsc, [mk/2, ref/1]).
 -import(emqx_dashboard_swagger, [error_codes/2]).
@@ -590,7 +591,7 @@ listener_authenticator(delete,
 authenticator_move(post,
                    #{bindings := #{id := AuthenticatorID},
                      body := #{<<"position">> := Position}}) ->
-    move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position);
+    move_authenticator([authentication], ?GLOBAL, AuthenticatorID, Position);
 authenticator_move(post, #{bindings := #{id := _}, body := _}) ->
     serialize_error({missing_parameter, position}).
 
@@ -599,7 +600,7 @@ listener_authenticator_move(post,
                               body := #{<<"position">> := Position}}) ->
     with_listener(ListenerID,
                   fun(Type, Name, ChainName) ->
-                        move_authenitcator([listeners, Type, Name, authentication],
+                        move_authenticator([listeners, Type, Name, authentication],
                                            ChainName,
                                            AuthenticatorID,
                                            Position)
@@ -725,7 +726,9 @@ create_authenticator(ConfKeyPath, ChainName, Config) ->
             raw_config := AuthenticatorsConfig}} ->
             {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig),
             {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))};
-        {error, {_, _, Reason}} ->
+        {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
+            serialize_error(Reason);
+        {error, Reason} ->
             serialize_error(Reason)
     end.
 
@@ -753,7 +756,9 @@ update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) ->
                raw_config := AuthenticatorsConfig}} ->
             {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig),
             {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))};
-        {error, {_, _, Reason}} ->
+        {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
+            serialize_error(Reason);
+        {error, Reason} ->
             serialize_error(Reason)
     end.
 
@@ -761,11 +766,13 @@ delete_authenticator(ConfKeyPath, ChainName, AuthenticatorID) ->
     case update_config(ConfKeyPath, {delete_authenticator, ChainName, AuthenticatorID}) of
         {ok, _} ->
             {204};
-        {error, {_, _, Reason}} ->
+        {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
+            serialize_error(Reason);
+        {error, Reason} ->
             serialize_error(Reason)
     end.
 
-move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) ->
+move_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Position) ->
     case parse_position(Position) of
         {ok, NPosition} ->
             case update_config(
@@ -773,7 +780,9 @@ move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) ->
                    {move_authenticator, ChainName, AuthenticatorID, NPosition}) of
                 {ok, _} ->
                     {204};
-                {error, {_, _, Reason}} ->
+                {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
+                    serialize_error(Reason);
+                {error, Reason} ->
                     serialize_error(Reason)
             end;
         {error, Reason} ->

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -160,13 +160,15 @@ authenticate(Credential, #{resource_id := ResourceId,
     Request = generate_request(Credential, State),
     case emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}) of
         {ok, 204, _Headers} -> {ok, #{is_superuser => false}};
+        {ok, 200, _Headers} -> {ok, #{is_superuser => false}};
         {ok, 200, Headers, Body} ->
             ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>),
             case safely_parse_body(ContentType, Body) of
                 {ok, NBody} ->
                     %% TODO: Return by user property
-                    {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false),
+                    UserProperty = maps:remove(<<"is_superuser">>, NBody),
-                           user_property => maps:remove(<<"is_superuser">>, NBody)}};
+                    IsSuperuser = emqx_authn_utils:is_superuser(NBody),
+                    {ok, IsSuperuser#{user_property => UserProperty}};
                 {error, _Reason} ->
                     {ok, #{is_superuser => false}}
             end;
@@ -208,9 +210,9 @@ check_url(URL) ->
     end.
 
 check_body(Body) ->
-    maps:fold(fun(_K, _V, false) -> false;
+    lists:all(
-                 (_K, V, true) -> is_binary(V)
+      fun erlang:is_binary/1,
-              end, true, Body).
+      maps:values(Body)).
 
 default_headers() ->
     maps:put(<<"content-type">>,
@@ -242,12 +244,9 @@ check_ssl_opts(Conf) ->
     end.
 
 check_headers(Conf) ->
-    Method = hocon_schema:get_value("config.method", Conf),
+    Method = to_bin(hocon_schema:get_value("config.method", Conf)),
     Headers = hocon_schema:get_value("config.headers", Conf),
-    case Method =:= get andalso maps:get(<<"content-type">>, Headers, undefined) =/= undefined of
+    Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)).
-        true -> false;
-        false -> true
-    end.
 
 parse_url(URL) ->
     {ok, URIMap} = emqx_http_lib:uri_parse(URL),
@@ -300,7 +299,7 @@ qs([], Acc) ->
     <<$&, Qs/binary>> = iolist_to_binary(lists:reverse(Acc)),
     Qs;
 qs([{K, V} | More], Acc) ->
-    qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]).
+    qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]).
 
 serialize_body(<<"application/json">>, Body) ->
     emqx_json:encode(Body);
@@ -327,7 +326,17 @@ may_append_body(Output, {ok, _, _, Body}) ->
 may_append_body(Output, {ok, _, _}) ->
     Output.
 
+uri_encode(T) ->
+    emqx_http_lib:uri_encode(to_bin(T)).
+
 to_list(A) when is_atom(A) ->
     atom_to_list(A);
 to_list(B) when is_binary(B) ->
     binary_to_list(B).
+
+to_bin(A) when is_atom(A) ->
+    atom_to_binary(A);
+to_bin(B) when is_binary(B) ->
+    B;
+to_bin(L) when is_list(L) ->
+    list_to_binary(L).

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -125,7 +125,7 @@ verify_claims(default) -> #{};
 verify_claims(validator) -> [fun do_check_verify_claims/1];
 verify_claims(converter) ->
     fun(VerifyClaims) ->
-        maps:to_list(VerifyClaims)
+        [{to_binary(K), V} || {K, V} <- maps:to_list(VerifyClaims)]
     end;
 verify_claims(_) -> undefined.
 
@@ -201,15 +201,14 @@ create2(#{use_jwks := false,
           secret := Secret0,
           secret_base64_encoded := Base64Encoded,
           verify_claims := VerifyClaims}) ->
-    Secret = case Base64Encoded of
+    case may_decode_secret(Base64Encoded, Secret0) of
-                 true ->
+        {error, Reason} ->
-                     base64:decode(Secret0);
+            {error, Reason};
-                 false ->
+        Secret ->
-                     Secret0
+            JWK = jose_jwk:from_oct(Secret),
-             end,
+            {ok, #{jwk => JWK,
-    JWK = jose_jwk:from_oct(Secret),
+                   verify_claims => VerifyClaims}}
-    {ok, #{jwk => JWK,
+    end;                                                                                           
-           verify_claims => VerifyClaims}};
 
 create2(#{use_jwks := false,
           algorithm := 'public-key',
@@ -234,6 +233,14 @@ create2(#{use_jwks := true,
             {error, Reason}
     end.
 
+may_decode_secret(false, Secret) -> Secret;
+may_decode_secret(true, Secret) ->
+    try base64:decode(Secret)
+    catch
+        error : _ ->
+            {error, {invalid_parameter, secret}}
+    end.
+
 replace_placeholder(L, Variables) ->
     replace_placeholder(L, Variables, []).
 
@@ -349,3 +356,8 @@ validate_placeholder(<<"clientid">>) ->
     clientid;
 validate_placeholder(<<"username">>) ->
     username.
+
+to_binary(A) when is_atom(A) ->
+    atom_to_binary(A);
+to_binary(B) when is_binary(B) ->
+    B.

apps/emqx_authn/test/emqx_authn_api_SUITE.erl

@@ -134,11 +134,23 @@ test_authenticators(PathPrefix) ->
                      uri(PathPrefix ++ ["authentication"]),
                      ValidConfig),
 
-    InvalidConfig = ValidConfig#{method => <<"delete">>},
+    {ok, 409, _} = request(
+                     post,
+                     uri(PathPrefix ++ ["authentication"]),
+                     ValidConfig),
+
+    InvalidConfig0 = ValidConfig#{method => <<"delete">>},
     {ok, 400, _} = request(
                      post,
                      uri(PathPrefix ++ ["authentication"]),
-                     InvalidConfig),
+                     InvalidConfig0),
+
+    InvalidConfig1 = ValidConfig#{method => <<"get">>,
+                                  headers => #{<<"content-type">> => <<"application/json">>}},
+    {ok, 400, _} = request(
+                     post,
+                     uri(PathPrefix ++ ["authentication"]),
+                     InvalidConfig1),
 
     ?assertAuthenticatorsMatch(
        [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}],
@@ -170,6 +182,13 @@ test_authenticator(PathPrefix) ->
                      uri(PathPrefix ++ ["authentication", "password-based:http"]),
                      InvalidConfig0),
 
+    InvalidConfig1 = ValidConfig0#{method => <<"get">>,
+                                  headers => #{<<"content-type">> => <<"application/json">>}},
+    {ok, 400, _} = request(
+                     put,
+                     uri(PathPrefix ++ ["authentication", "password-based:http"]),
+                     InvalidConfig1),
+
     ValidConfig1 = ValidConfig0#{pool_size => 9},
     {ok, 200, _} = request(
                      put,

apps/emqx_authn/test/emqx_authn_http_SUITE.erl

@@ -0,0 +1,392 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_http_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include("emqx_authn.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_placeholder.hrl").
+
+-define(PATH, [authentication]).
+
+-define(HTTP_PORT, 33333).
+-define(HTTP_PATH, "/auth").
+-define(CREDENTIALS, #{username => <<"plain">>,
+                       password => <<"plain">>,
+                       listener => 'tcp:default',
+                       protocol => mqtt
+                      }).
+
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_common_test_helpers:start_apps([emqx_authn]),
+    application:ensure_all_started(cowboy),
+    Config.
+
+end_per_suite(_) ->
+    emqx_authn_test_lib:delete_authenticators(
+      [authentication],
+      ?GLOBAL),
+    emqx_common_test_helpers:stop_apps([emqx_authn]),
+    application:stop(cowboy),
+    ok.
+
+init_per_testcase(_Case, Config) ->
+    emqx_authn_test_lib:delete_authenticators(
+      [authentication],
+      ?GLOBAL),
+    emqx_authn_http_test_server:start(?HTTP_PORT, ?HTTP_PATH),
+    Config.
+
+end_per_testcase(_Case, _Config) ->
+    ok = emqx_authn_http_test_server:stop() .
+
+%%------------------------------------------------------------------------------
+%% Tests
+%%------------------------------------------------------------------------------
+
+t_create(_Config) ->
+    AuthConfig = raw_http_auth_config(),
+
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {create_authenticator, ?GLOBAL, AuthConfig}),
+
+    {ok, [#{provider := emqx_authn_http}]} = emqx_authentication:list_authenticators(?GLOBAL).
+
+t_create_invalid(_Config) ->
+    AuthConfig = raw_http_auth_config(),
+
+    InvalidConfigs =
+        [
+         AuthConfig#{headers => []},
+         AuthConfig#{method => delete}
+        ],
+
+    lists:foreach(
+      fun(Config) ->
+              ct:pal("creating authenticator with invalid config: ~p", [Config]),
+              {error, _} =
+                  try
+                      emqx:update_config(
+                        ?PATH,
+                        {create_authenticator, ?GLOBAL, Config})
+                  catch
+                      throw:Error ->
+                          {error, Error}
+                  end,
+              {ok, []} = emqx_authentication:list_authenticators(?GLOBAL)
+      end,
+      InvalidConfigs).
+
+t_authenticate(_Config) ->
+    ok = lists:foreach(
+           fun(Sample) ->
+                   ct:pal("test_user_auth sample: ~p", [Sample]),
+                   test_user_auth(Sample)
+           end,
+           samples()).
+
+test_user_auth(#{handler := Handler,
+                 config_params := SpecificConfgParams,
+                 result := Result}) ->
+    AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams),
+
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {create_authenticator, ?GLOBAL, AuthConfig}),
+
+    emqx_authn_http_test_server:set_handler(Handler),
+
+    ?assertEqual(Result, emqx_access_control:authenticate(?CREDENTIALS)),
+
+    emqx_authn_test_lib:delete_authenticators(
+      [authentication],
+      ?GLOBAL).
+
+t_destroy(_Config) ->
+    AuthConfig = raw_http_auth_config(),
+
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {create_authenticator, ?GLOBAL, AuthConfig}),
+
+    ok = emqx_authn_http_test_server:set_handler(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end),
+
+    {ok, [#{provider := emqx_authn_http, state := State}]}
+        = emqx_authentication:list_authenticators(?GLOBAL),
+
+    Credentials = maps:with([username, password], ?CREDENTIALS),
+
+    {ok, _} = emqx_authn_http:authenticate(
+                Credentials,
+                State),
+
+    emqx_authn_test_lib:delete_authenticators(
+      [authentication],
+      ?GLOBAL),
+
+    % Authenticator should not be usable anymore
+    ?assertException(
+       error,
+       _,
+       emqx_authn_http:authenticate(
+         Credentials,
+         State)).
+
+t_update(_Config) ->
+    CorrectConfig = raw_http_auth_config(),
+    IncorrectConfig =
+        CorrectConfig#{url => <<"http://127.0.0.1:33333/invalid">>},
+
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {create_authenticator, ?GLOBAL, IncorrectConfig}),
+
+    ok = emqx_authn_http_test_server:set_handler(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end),
+
+    {error, not_authorized} = emqx_access_control:authenticate(?CREDENTIALS),
+
+    % We update with config with correct query, provider should update and work properly
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {update_authenticator, ?GLOBAL, <<"password-based:http">>, CorrectConfig}),
+
+    {ok,_} = emqx_access_control:authenticate(?CREDENTIALS).
+
+t_is_superuser(_Config) ->
+    Config = raw_http_auth_config(),
+    {ok, _} = emqx:update_config(
+                ?PATH,
+                {create_authenticator, ?GLOBAL, Config}),
+
+    Checks = [
+              {json, <<"0">>,  false},
+              {json, <<"">>,   false},
+              {json, null,     false},
+              {json, 0,        false},
+
+              {json, <<"1">>,   true},
+              {json, <<"val">>, true},
+              {json, 1,         true},
+              {json, 123,       true},
+
+              {form, <<"0">>,  false},
+              {form, <<"">>,   false},
+
+              {form, <<"1">>,   true},
+              {form, <<"val">>, true}
+             ],
+
+    lists:foreach(fun test_is_superuser/1, Checks).
+
+test_is_superuser({Kind, Value, ExpectedValue}) ->
+
+    {ContentType, Res} = case Kind of
+                             json ->
+                                 {<<"application/json">>,
+                                  jiffy:encode(#{is_superuser => Value})};
+                             form ->
+                                 {<<"application/x-www-form-urlencoded">>,
+                                  iolist_to_binary([<<"is_superuser=">>, Value])}
+                         end,
+
+    emqx_authn_http_test_server:set_handler(
+      fun(Req0, State) ->
+              Req = cowboy_req:reply(
+                      200,
+                      #{<<"content-type">> => ContentType},
+                      Res,
+                      Req0),
+              {ok, Req, State}
+      end),
+
+    ?assertMatch(
+       {ok, #{is_superuser := ExpectedValue}},
+       emqx_access_control:authenticate(?CREDENTIALS)).
+
+%%------------------------------------------------------------------------------
+%% Helpers
+%%------------------------------------------------------------------------------
+
+raw_http_auth_config() ->
+    #{
+        mechanism => <<"password-based">>,
+        enable => <<"true">>,
+
+        backend => <<"http">>,
+        method => <<"get">>,
+        url => <<"http://127.0.0.1:33333/auth">>,
+        body => #{<<"username">> => ?PH_USERNAME, <<"password">> => ?PH_PASSWORD},
+        headers => #{<<"X-Test-Header">> => <<"Test Value">>}
+    }.
+
+samples() ->
+    [
+     %% simple get request
+     #{handler => fun(Req0, State) ->
+                          #{username := <<"plain">>,
+                            password := <<"plain">>
+                           } = cowboy_req:match_qs([username, password], Req0),
+
+                          Req = cowboy_req:reply(200, Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => false}}
+      },
+
+     %% get request with json body response
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(
+                                  200,
+                                  #{<<"content-type">> => <<"application/json">>},
+                                  jiffy:encode(#{is_superuser => true}),
+                                  Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => true, user_property => #{}}}
+      },
+
+     %% get request with url-form-encoded body response
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(
+                                  200,
+                                  #{<<"content-type">> =>
+                                    <<"application/x-www-form-urlencoded">>},
+                                  <<"is_superuser=true">>,
+                                  Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => true, user_property => #{}}}
+      },
+
+     %% get request with response of unknown encoding
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(
+                                  200,
+                                  #{<<"content-type">> =>
+                                    <<"test/plain">>},
+                                  <<"is_superuser=true">>,
+                                  Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => false}}
+      },
+
+     %% simple post request, application/json
+     #{handler => fun(Req0, State) ->
+                          {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+                          #{<<"username">> := <<"plain">>,
+                            <<"password">> := <<"plain">>
+                           } = jiffy:decode(RawBody, [return_maps]),
+                          Req = cowboy_req:reply(200, Req1),
+                          {ok, Req, State}
+                  end,
+       config_params => #{
+                          method => post,
+                          headers => #{<<"content-type">> => <<"application/json">>}
+                         },
+       result => {ok,#{is_superuser => false}}
+      },
+
+     %% simple post request, application/x-www-form-urlencoded
+     #{handler => fun(Req0, State) ->
+                          {ok, PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0),
+                          #{<<"username">> := <<"plain">>,
+                            <<"password">> := <<"plain">>
+                           } = maps:from_list(PostVars),
+                          Req = cowboy_req:reply(200, Req1),
+                          {ok, Req, State}
+                  end,
+       config_params => #{
+                          method => post,
+                          headers => #{<<"content-type">> =>
+                                       <<"application/x-www-form-urlencoded">>}
+                         },
+       result => {ok,#{is_superuser => false}}
+      }
+
+     %% 204 code
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(204, Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => false}}
+      },
+
+     %% custom headers
+     #{handler => fun(Req0, State) ->
+                          <<"Test Value">> = cowboy_req:header(<<"x-test-header">>, Req0),
+                          Req = cowboy_req:reply(200, Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {ok,#{is_superuser => false}}
+      },
+
+     %% 400 code
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(400, Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {error,not_authorized}
+      },
+
+     %% 500 code
+     #{handler => fun(Req0, State) ->
+                          Req = cowboy_req:reply(500, Req0),
+                          {ok, Req, State}
+                  end,
+       config_params => #{},
+       result => {error,not_authorized}
+      },
+
+     %% Handling error
+     #{handler => fun(Req0, State) ->
+                          error(woops),
+                          {ok, Req0, State}
+                  end,
+       config_params => #{},
+       result => {error,not_authorized}
+      }
+    ].
+
+start_apps(Apps) ->
+    lists:foreach(fun application:ensure_all_started/1, Apps).
+
+stop_apps(Apps) ->
+    lists:foreach(fun application:stop/1, Apps).

apps/emqx_authn/test/emqx_authn_http_test_server.erl

@@ -0,0 +1,89 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_http_test_server).
+
+-behaviour(gen_server).
+-behaviour(cowboy_handler).
+
+% cowboy_server callbacks
+-export([init/2]).
+
+% gen_server callbacks
+-export([init/1,
+         handle_call/3,
+         handle_cast/2
+        ]).
+
+% API
+-export([start/2,
+         stop/0,
+         set_handler/1
+        ]).
+
+%%------------------------------------------------------------------------------
+%% API
+%%------------------------------------------------------------------------------
+
+start(Port, Path) ->
+    Dispatch = cowboy_router:compile([
+        {'_', [{Path, ?MODULE, []}]}
+    ]),
+    {ok, _} = cowboy:start_clear(?MODULE,
+        [{port, Port}],
+        #{env => #{dispatch => Dispatch}}
+    ),
+    {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []),
+    ok.
+
+stop() ->
+    gen_server:stop(?MODULE),
+    cowboy:stop_listener(?MODULE).
+
+set_handler(F) when is_function(F, 2) ->
+    gen_server:call(?MODULE, {set_handler, F}).
+
+%%------------------------------------------------------------------------------
+%% gen_server API
+%%------------------------------------------------------------------------------
+
+init([]) ->
+    F = fun(Req0, State) ->
+                Req = cowboy_req:reply(
+                        400,
+                        #{<<"content-type">> => <<"text/plain">>},
+                        <<"">>,
+                        Req0),
+                {ok, Req, State}
+        end,
+    {ok, F}.
+
+handle_cast(_, F) ->
+    {noreply, F}.
+
+handle_call({set_handler, F}, _From, _F) ->
+    {reply, ok, F};
+
+handle_call(get_handler, _From, F) ->
+    {reply, F, F}.
+
+%%------------------------------------------------------------------------------
+%% cowboy_server API
+%%------------------------------------------------------------------------------
+
+init(Req, State) ->
+    Handler = gen_server:call(?MODULE, get_handler),
+    Handler(Req, State).

apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl

@@ -19,140 +19,142 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
-% -include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct.hrl").
-% -include_lib("eunit/include/eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
 
-% -include("emqx_authn.hrl").
+-include("emqx_authn.hrl").
 
-% -define(AUTH, emqx_authn).
+-define(AUTHN_ID, <<"mechanism:jwt">>).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
-% init_per_suite(Config) ->
+init_per_suite(Config) ->
-%     emqx_common_test_helpers:start_apps([emqx_authn]),
+    emqx_common_test_helpers:start_apps([emqx_authn]),
-%     Config.
+    Config.
 
-% end_per_suite(_) ->
+end_per_suite(_) ->
-%     emqx_common_test_helpers:stop_apps([emqx_authn]),
+    emqx_common_test_helpers:stop_apps([emqx_authn]),
-%     ok.
+    ok.
 
-% t_jwt_authenticator(_) ->
+t_jwt_authenticator(_) ->
-%     AuthenticatorName = <<"myauthenticator">>,
+    Secret = <<"abcdef">>,
-%     Config = #{name => AuthenticatorName,
+    Config = #{mechanism => jwt,
-%                mechanism => jwt,
+               use_jwks => false,
-%                use_jwks => false,
+               algorithm => 'hmac-based',
-%                algorithm => 'hmac-based',
+               secret => Secret,
-%                secret => <<"abcdef">>,
+               secret_base64_encoded => false,
-%                secret_base64_encoded => false,
+               verify_claims => []},
-%                verify_claims => []},
+    {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
-%     {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config),
 
-%     Payload = #{<<"username">> => <<"myuser">>},
+    Payload = #{<<"username">> => <<"myuser">>},
-%     JWS = generate_jws('hmac-based', Payload, <<"abcdef">>),
+    JWS = generate_jws('hmac-based', Payload, Secret),
-%     ClientInfo = #{username => <<"myuser">>,
+    Credential = #{username => <<"myuser">>,
-% 			       password => JWS},
+			       password => JWS},
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)),
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)),
 
-%     Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true},
+    Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true},
-%     JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>),
+    JWS1 = generate_jws('hmac-based', Payload1, Secret),
-%     ClientInfo1 = #{username => <<"myuser">>,
+    Credential1 = #{username => <<"myuser">>,
-% 			        password => JWS1},
+			        password => JWS1},
-%     ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)),
+    ?assertEqual({ok, #{is_superuser => true}}, emqx_authn_jwt:authenticate(Credential1, State)),
 
-%     BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>),
+    BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>),
-%     ClientInfo2 = ClientInfo#{password => BadJWS},
+    Credential2 = Credential#{password => BadJWS},
-%     ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)),
+    ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential2, State)),
 
-%     %% secret_base64_encoded
+    %% secret_base64_encoded
-%     Config2 = Config#{secret => base64:encode(<<"abcdef">>),
+    Config2 = Config#{secret => base64:encode(Secret),
-%                       secret_base64_encoded => true},
+                      secret_base64_encoded => true},
-%     ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)),
+    {ok, State2} = emqx_authn_jwt:update(Config2, State),
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)),
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State2)),
 
-%     Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]},
+    %% invalid secret
-%     ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)),
+    BadConfig = Config#{secret => <<"emqxsecret">>,
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)),
+                        secret_base64_encoded => true},
-%     ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)),
+    {error, {invalid_parameter, secret}} = emqx_authn_jwt:create(?AUTHN_ID, BadConfig),
 
-%     %% Expiration
+    Config3 = Config#{verify_claims => [{<<"username">>, <<"${username}">>}]},
-%     Payload3 = #{ <<"username">> => <<"myuser">>
+    {ok, State3} = emqx_authn_jwt:update(Config3, State2),
-%                 , <<"exp">> => erlang:system_time(second) - 60},
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State3)),
-%     JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>),
+    ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential#{username => <<"otheruser">>}, State3)),
-%     ClientInfo3 = ClientInfo#{password => JWS3},
-%     ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)),
 
-%     Payload4 = #{ <<"username">> => <<"myuser">>
+    %% Expiration
-%                 , <<"exp">> => erlang:system_time(second) + 60},
+    Payload3 = #{ <<"username">> => <<"myuser">>
-%     JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>),
+                , <<"exp">> => erlang:system_time(second) - 60},
-%     ClientInfo4 = ClientInfo#{password => JWS4},
+    JWS3 = generate_jws('hmac-based', Payload3, Secret),
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)),
+    Credential3 = Credential#{password => JWS3},
+    ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential3, State3)),
 
-%     %% Issued At
+    Payload4 = #{ <<"username">> => <<"myuser">>
-%     Payload5 = #{ <<"username">> => <<"myuser">>
+                , <<"exp">> => erlang:system_time(second) + 60},
-%                 , <<"iat">> => erlang:system_time(second) - 60},
+    JWS4 = generate_jws('hmac-based', Payload4, Secret),
-%     JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>),
+    Credential4 = Credential#{password => JWS4},
-%     ClientInfo5 = ClientInfo#{password => JWS5},
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential4, State3)),
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)),
 
-%     Payload6 = #{ <<"username">> => <<"myuser">>
+    %% Issued At
-%                 , <<"iat">> => erlang:system_time(second) + 60},
+    Payload5 = #{ <<"username">> => <<"myuser">>
-%     JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>),
+                , <<"iat">> => erlang:system_time(second) - 60},
-%     ClientInfo6 = ClientInfo#{password => JWS6},
+    JWS5 = generate_jws('hmac-based', Payload5, Secret),
-%     ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)),
+    Credential5 = Credential#{password => JWS5},
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential5, State3)),
 
-%     %% Not Before
+    Payload6 = #{ <<"username">> => <<"myuser">>
-%     Payload7 = #{ <<"username">> => <<"myuser">>
+                , <<"iat">> => erlang:system_time(second) + 60},
-%                 , <<"nbf">> => erlang:system_time(second) - 60},
+    JWS6 = generate_jws('hmac-based', Payload6, Secret),
-%     JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>),
+    Credential6 = Credential#{password => JWS6},
-%     ClientInfo7 = ClientInfo#{password => JWS7},
+    ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential6, State3)),
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)),
 
-%     Payload8 = #{ <<"username">> => <<"myuser">>
+    %% Not Before
-%                 , <<"nbf">> => erlang:system_time(second) + 60},
+    Payload7 = #{ <<"username">> => <<"myuser">>
-%     JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>),
+                , <<"nbf">> => erlang:system_time(second) - 60},
-%     ClientInfo8 = ClientInfo#{password => JWS8},
+    JWS7 = generate_jws('hmac-based', Payload7, Secret),
-%     ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)),
+    Credential7 = Credential6#{password => JWS7},
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential7, State3)),
 
-%     ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)),
+    Payload8 = #{ <<"username">> => <<"myuser">>
-%     ok.
+                , <<"nbf">> => erlang:system_time(second) + 60},
+    JWS8 = generate_jws('hmac-based', Payload8, Secret),
+    Credential8 = Credential#{password => JWS8},
+    ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential8, State3)),
 
-% t_jwt_authenticator2(_) ->
+    ?assertEqual(ok, emqx_authn_jwt:destroy(State3)),
-%     Dir = code:lib_dir(emqx_authn, test),
+    ok.
-%     PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])),
-%     PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])),
-%     AuthenticatorName = <<"myauthenticator">>,
-%     Config = #{name => AuthenticatorName,
-%                mechanism => jwt,
-%                use_jwks => false,
-%                algorithm => 'public-key',
-%                certificate => PublicKey,
-%                verify_claims => []},
-%     {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config),
 
-%     Payload = #{<<"username">> => <<"myuser">>},
+t_jwt_authenticator2(_) ->
-%     JWS = generate_jws('public-key', Payload, PrivateKey),
+    Dir = code:lib_dir(emqx_authn, test),
-%     ClientInfo = #{username => <<"myuser">>,
+    PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])),
-% 			       password => JWS},
+    PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])),
-%     ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)),
+    Config = #{mechanism => jwt,
-%     ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)),
+               use_jwks => false,
+               algorithm => 'public-key',
+               certificate => PublicKey,
+               verify_claims => []},
+    {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 
-%     ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)),
+    Payload = #{<<"username">> => <<"myuser">>},
-%     ok.
+    JWS = generate_jws('public-key', Payload, PrivateKey),
+    Credential = #{username => <<"myuser">>,
+			       password => JWS},
+    ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)),
+    ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State)),
 
-% generate_jws('hmac-based', Payload, Secret) ->
+    ?assertEqual(ok, emqx_authn_jwt:destroy(State)),
-%     JWK = jose_jwk:from_oct(Secret),
+    ok.
-%     Header = #{ <<"alg">> => <<"HS256">>
+
-%               , <<"typ">> => <<"JWT">>
+generate_jws('hmac-based', Payload, Secret) ->
-%               },
+    JWK = jose_jwk:from_oct(Secret),
-%     Signed = jose_jwt:sign(JWK, Header, Payload),
+    Header = #{ <<"alg">> => <<"HS256">>
-%     {_, JWS} = jose_jws:compact(Signed),
+              , <<"typ">> => <<"JWT">>
-%     JWS;
+              },
-% generate_jws('public-key', Payload, PrivateKey) ->
+    Signed = jose_jwt:sign(JWK, Header, Payload),
-%     JWK = jose_jwk:from_pem_file(PrivateKey),
+    {_, JWS} = jose_jws:compact(Signed),
-%     Header = #{ <<"alg">> => <<"RS256">>
+    JWS;
-%               , <<"typ">> => <<"JWT">>
+generate_jws('public-key', Payload, PrivateKey) ->
-%               },
+    JWK = jose_jwk:from_pem_file(PrivateKey),
-%     Signed = jose_jwt:sign(JWK, Header, Payload),
+    Header = #{ <<"alg">> => <<"RS256">>
-%     {_, JWS} = jose_jws:compact(Signed),
+              , <<"typ">> => <<"JWT">>
-%     JWS.
+              },
+    Signed = jose_jwt:sign(JWK, Header, Payload),
+    {_, JWS} = jose_jws:compact(Signed),
+    JWS.

apps/emqx_authz/include/emqx_authz.hrl

@@ -68,6 +68,8 @@
 
 -define(CONF_KEY_PATH, [authorization, sources]).
 
+-define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}").
+
 -define(USERNAME_RULES_EXAMPLE, #{username => user1,
                                   rules => [ #{topic => <<"test/toopic/1">>,
                                                permission => <<"allow">>,

apps/emqx_authz/src/emqx_authz_mysql.erl

@@ -38,10 +38,10 @@ description() ->
 parse_query(undefined) ->
     undefined;
 parse_query(Sql) ->
-    case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of
+    case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
         {match, Variables} ->
             Params = [Var || [Var] <- Variables],
-            {re:replace(Sql, "'%[ucCad]'", "?", [global, {return, list}]), Params};
+            {re:replace(Sql, ?RE_PLACEHOLDER, "?", [global, {return, list}]), Params};
         nomatch ->
             {Sql, []}
     end.

apps/emqx_authz/src/emqx_authz_postgresql.erl

@@ -38,7 +38,7 @@ description() ->
 parse_query(undefined) ->
     undefined;
 parse_query(Sql) ->
-    case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of
+    case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
         {match, Variables} ->
             Params = [Var || [Var] <- Variables],
             Vars = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Params))],

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -169,8 +169,11 @@ health_check(PoolName) ->
         case ecpool_worker:client(Worker) of
             {ok, Conn} ->
                 %% we don't care if this returns something or not, we just to test the connection
-                Res = mongo_api:find_one(Conn, <<"foo">>, {}, #{}),
+                try mongo_api:find_one(Conn, <<"foo">>, {}, #{}) of
-                Res == undefined orelse is_map(Res);
+                    _ -> true
+                catch
+                    _Class:_Error -> false
+                end;
             _ -> false
         end
     end || {_WorkerName, Worker} <- ecpool:workers(PoolName)],

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -31,7 +31,8 @@
 -define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>,
                                                          ?TO_REF(namespace(_M_), _F_)])).
 
--define(MAX_ROW_LIMIT, 100).
+-define(MAX_ROW_LIMIT, 1000).
+-define(DEFAULT_ROW, 100).
 
 -type(request() :: #{bindings => map(), query_string => map(), body => map()}).
 -type(request_meta() :: #{module => module(), path => string(), method => atom()}).
@@ -80,7 +81,7 @@ fields(page) ->
 fields(limit) ->
     Desc = iolist_to_binary([<<"Results per page(max ">>,
         integer_to_binary(?MAX_ROW_LIMIT), <<")">>]),
-    Meta = #{in => query, desc => Desc, default => ?MAX_ROW_LIMIT, example => 50},
+    Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50},
     [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}].
 
 -spec(schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map()).
@@ -123,10 +124,10 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
         {Bindings, QueryStr} = check_parameters(Request, Params, Module),
         NewBody = check_request_body(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
         {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}}
-    catch throw:Error ->
+    catch throw:{_, ValidErrors} ->
-        {_, [{validation_error, ValidErr}]} = Error,
+        Msg = [io_lib:format("~ts : ~p", [Key, Reason]) ||
-        #{path := Key, reason := Reason} = ValidErr,
+            {validation_error, #{path := Key, reason := Reason}} <- ValidErrors],
-        {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~ts : ~p", [Key, Reason]))}
+        {400, 'BAD_REQUEST', iolist_to_binary(string:join(Msg, ","))}
     end.
 
 check_and_translate(Schema, Map, Opts) ->

apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl

@@ -71,8 +71,10 @@ t_public_ref(_Config) ->
         {emqx_dashboard_swagger, page, parameter}
     ], Refs),
     ExpectRefs = [
-        #{<<"public.limit">> => #{description => <<"Results per page(max 100)">>, example => 50,in => query,name => limit,
+        #{<<"public.limit">> => #{description => <<"Results per page(max 1000)">>,
-            schema => #{default => 100,example => 1,maximum => 100, minimum => 1,type => integer}}},
+            example => 50,in => query,name => limit,
+            schema => #{default => 100,example => 1,maximum => 1000,
+                minimum => 1,type => integer}}},
         #{<<"public.page">> => #{description => <<"Page number of the results to fetch.">>,
             example => 1,in => query,name => page,
             schema => #{default => 1,example => 100,type => integer}}}],
@@ -176,7 +178,8 @@ t_in_mix_trans(_Config) ->
     Expect = {ok,
         #{body => #{},
             bindings => #{state => 720},
-            query_string => #{<<"filter">> => created,<<"is_admin">> => true, <<"per_page">> => 5,<<"timeout">> => 34}}},
+            query_string => #{<<"filter">> => created,<<"is_admin">> => true,
+                <<"per_page">> => 5,<<"timeout">> => 34}}},
     ?assertEqual(Expect, trans_parameters(Path, Bindings, Query)),
     ok.
 
@@ -268,7 +271,10 @@ schema("/test/in/:filter") ->
             parameters => [
                 {filter,
                     mk(hoconsc:enum([assigned, created, mentioned, all]),
-                        #{in => path, desc => <<"Indicates which sorts of issues to return">>, example => "all"})}
+                        #{in => path,
+                          desc => <<"Indicates which sorts of issues to return">>,
+                          example => "all"
+                        })}
             ],
             responses => #{200 => <<"ok">>}
         }
@@ -323,9 +329,11 @@ schema("/test/in/mix/:state") ->
             deprecated => true,
             parameters => [
                 {filter, hoconsc:mk(hoconsc:enum([assigned, created, mentioned, all]),
-                    #{in => query, desc => <<"Indicates which sorts of issues to return">>, example => "all"})},
+                    #{in => query, desc => <<"Indicates which sorts of issues to return">>,
+                        example => "all"})},
                 {state, mk(emqx_schema:duration_s(),
-                    #{in => path, required => true, example => "12m", desc => <<"Indicates the state of the issues to return.">>})},
+                    #{in => path, required => true, example => "12m",
+                        desc => <<"Indicates the state of the issues to return.">>})},
                 {per_page, mk(range(1, 50),
                     #{in => query, required => false, example => 10, default => 5})},
                 {is_admin, mk(boolean(), #{in => query})},

apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl

@@ -47,9 +47,14 @@ t_object(_Config) ->
             #{<<"schema">> =>
             #{required => [<<"timeout">>, <<"per_page">>],
                 <<"properties">> =>[
-                    {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+                    {<<"per_page">>, #{description => <<"good per page desc">>,
-                    {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
+                        example => 1, maximum => 100, minimum => 1, type => integer}},
-                    {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
+                    {<<"timeout">>, #{default => 5, <<"oneOf">> =>
+                    [#{example => <<"1h">>, type => string},
+                        #{enum => [infinity], type => string}]}},
+                    {<<"inner_ref">>,
+                        #{<<"$ref">> =>
+                        <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
                 <<"type">> => object}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, good_ref}],
@@ -63,14 +68,20 @@ t_nest_object(_Config) ->
             #{<<"schema">> =>
             #{required => [<<"timeout">>],
                 <<"properties">> =>
-                [{<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+                [{<<"per_page">>, #{description => <<"good per page desc">>,
+                    example => 1, maximum => 100, minimum => 1, type => integer}},
                     {<<"timeout">>, #{default => 5, <<"oneOf">> =>
-                    [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
+                    [#{example => <<"1h">>, type => string},
+                        #{enum => [infinity], type => string}]}},
                     {<<"nest_object">>,
                         #{<<"properties">> =>
                         [{<<"good_nest_1">>, #{example => 100, type => integer}},
-                            {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],<<"type">> => object}},
+                            {<<"good_nest_2">>, #{<<"$ref">> =>
-                    {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
+                            <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
+                            <<"type">> => object}},
+                    {<<"inner_ref">>,
+                        #{<<"$ref">> =>
+                        <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
                 <<"type">> => object}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, good_ref}],
@@ -81,7 +92,8 @@ t_local_ref(_Config) ->
     Spec = #{
         post => #{parameters => [],
             requestBody => #{<<"content">> => #{<<"application/json">> =>
-            #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}}},
+            #{<<"schema">> => #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, good_ref}],
     validate("/ref/local", Spec, Refs),
@@ -91,17 +103,22 @@ t_remote_ref(_Config) ->
     Spec = #{
         post => #{parameters => [],
             requestBody => #{<<"content">> => #{<<"application/json">> =>
-            #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}}}},
+            #{<<"schema">> => #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{emqx_swagger_remote_schema, "ref2"}],
     {_, Components} = validate("/ref/remote", Spec, Refs),
     ExpectComponents = [
         #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"properties">> => [
-            {<<"page">>, #{description => <<"good page">>,example => 1, maximum => 100,minimum => 1,type => integer}},
+            {<<"page">>, #{description => <<"good page">>,example => 1,
-        {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}},
+                maximum => 100,minimum => 1,type => integer}},
+        {<<"another_ref">>, #{<<"$ref">> =>
+        <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}},
         #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"properties">> => [
-            {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}},
+            {<<"ip">>, #{description => <<"IP:Port">>,
-            {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>,type => string}}],
+                example => <<"127.0.0.1:80">>,type => string}},
+            {<<"version">>, #{description => <<"a good version">>,
+                example => <<"1.0.0">>,type => string}}],
             <<"type">> => object}}],
     ?assertEqual(ExpectComponents, Components),
     ok.
@@ -110,18 +127,22 @@ t_nest_ref(_Config) ->
     Spec = #{
         post => #{parameters => [],
             requestBody => #{<<"content">> => #{<<"application/json">> =>
-            #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.nest_ref">>}}}},
+            #{<<"schema">> => #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_requestBody_SUITE.nest_ref">>}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, nest_ref}],
     ExpectComponents = lists:sort([
         #{<<"emqx_swagger_requestBody_SUITE.nest_ref">> => #{<<"properties">> => [
             {<<"env">>, #{enum => [test,dev,prod],type => string}},
-            {<<"another_ref">>, #{description => <<"nest ref">>, <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
+            {<<"another_ref">>, #{description => <<"nest ref">>,
+                <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
             <<"type">> => object}},
         #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [
-            {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}},
+            {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>,
+                example => <<"127.0.0.1:80">>,type => string}},
             {<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}},
-            {<<"tag">>, #{description => <<"tag">>, example => <<"binary-example">>,type => string}}],
+            {<<"tag">>, #{description => <<"tag">>,
+                example => <<"binary-example">>,type => string}}],
             <<"type">> => object}}]),
     {_, Components} = validate("/ref/nest/ref", Spec, Refs),
     ?assertEqual(ExpectComponents, Components),
@@ -153,9 +174,14 @@ t_ref_array_with_key(_Config) ->
             #{<<"schema">> => #{required => [<<"timeout">>],
                 <<"type">> => object, <<"properties">> =>
                 [
-                    {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+                    {<<"per_page">>, #{description => <<"good per page desc">>,
-                    {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
+                        example => 1, maximum => 100, minimum => 1, type => integer}},
-                    {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}
+                    {<<"timeout">>, #{default => 5, <<"oneOf">> =>
+                    [#{example => <<"1h">>, type => string},
+                        #{enum => [infinity], type => string}]}},
+                    {<<"array_refs">>, #{items => #{<<"$ref">> =>
+                    <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>},
+                        type => array}}
                 ]}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, good_ref}],
@@ -166,7 +192,8 @@ t_ref_array_without_key(_Config) ->
     Spec = #{
         post => #{parameters => [],
             requestBody => #{<<"content">> => #{<<"application/json">> => #{<<"schema">> =>
-            #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}}},
+            #{items => #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}}},
             responses => #{<<"200">> => #{description => <<"ok">>}}}},
     Refs = [{?MODULE, good_ref}],
     validate("/ref/array/without/key", Spec, Refs),
@@ -190,7 +217,8 @@ t_api_spec(_Config) ->
         {ok, #{body := #{<<"timeout">> := <<"infinity">>}}},
         trans_requestBody(Path, Body, Filter0)),
 
-    {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}),
+    {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE,
+        #{check_schema => true, translate_body => true}),
     Filter1 = filter(Spec1, Path),
     ?assertMatch(
         {ok, #{body := #{<<"timeout">> := infinity}}},
@@ -237,7 +265,8 @@ t_object_notrans(_Config) ->
             <<"tag">> => <<"god_tag">>
         }
     },
-    {ok, #{body := ActualBody}} = trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request/2),
+    {ok, #{body := ActualBody}} = trans_requestBody(Path, Body,
+        fun emqx_dashboard_swagger:filter_check_request/2),
     ?assertEqual(Body, ActualBody),
     ok.
 
@@ -444,7 +473,8 @@ filter(ApiSpec, Path) ->
     Filter.
 
 trans_requestBody(Path, Body) ->
-    trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2).
+    trans_requestBody(Path, Body,
+        fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2).
 
 trans_requestBody(Path, Body, Filter) ->
     Meta = #{module => ?MODULE, method => post, path => Path},
@@ -453,7 +483,8 @@ trans_requestBody(Path, Body, Filter) ->
 
 api_spec() -> emqx_dashboard_swagger:spec(?MODULE).
 paths() ->
-    ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/ref/array/with/key", "/ref/array/without/key"].
+    ["/object", "/nest/object", "/ref/local", "/ref/nest/ref",
+        "/ref/array/with/key", "/ref/array/without/key"].
 
 schema("/object") ->
     to_schema([

apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl

@@ -40,9 +40,12 @@ t_object(_config) ->
         #{<<"content">> => #{<<"application/json">> =>
         #{<<"schema">> => #{required => [<<"timeout">>, <<"per_page">>],
             <<"properties">> => [
-                {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+                {<<"per_page">>, #{description => <<"good per page desc">>,
-                {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
+                    example => 1, maximum => 100, minimum => 1, type => integer}},
-                {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}],
+                {<<"timeout">>, #{default => 5, <<"oneOf">> =>
+                [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
+                {<<"inner_ref">>, #{<<"$ref">> =>
+                <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}],
             <<"type">> => object}}}},
     ExpectRefs = [{?MODULE, good_ref}],
     validate(Path, Object, ExpectRefs),
@@ -80,14 +83,17 @@ t_nest_object(_Config) ->
     Object =
         #{<<"content">> => #{<<"application/json">> => #{<<"schema">> =>
         #{required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [
-            {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+            {<<"per_page">>, #{description => <<"good per page desc">>, example => 1,
+                maximum => 100, minimum => 1, type => integer}},
             {<<"timeout">>, #{default => 5, <<"oneOf">> =>
             [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
             {<<"nest_object">>, #{<<"type">> => object, <<"properties">> => [
                 {<<"good_nest_1">>, #{example => 100, type => integer}},
-                {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}
+                {<<"good_nest_2">>, #{<<"$ref">> =>
+                <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}
                 }]}},
-            {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}]
+            {<<"inner_ref">>, #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}]
         }}}},
     ExpectRefs = [{?MODULE, good_ref}],
     validate(Path, Object, ExpectRefs),
@@ -138,7 +144,8 @@ t_bad_ref(_Config) ->
     Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> =>
     #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.bad_ref">>}}}},
     ExpectRefs = [{?MODULE, bad_ref}],
-    ?assertThrow({error, #{module := ?MODULE, msg := <<"Object only supports not empty proplists">>}},
+    ?assertThrow({error, #{module := ?MODULE,
+        msg := <<"Object only supports not empty proplists">>}},
         validate(Path, Object, ExpectRefs)),
     ok.
 
@@ -158,7 +165,8 @@ t_nest_ref(_Config) ->
 
 t_complicated_type(_Config) ->
     Path = "/ref/complicated_type",
-    Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{<<"properties">> =>
+    Object = #{<<"content">> => #{<<"application/json">> =>
+    #{<<"schema">> => #{<<"properties">> =>
     [
         {<<"no_neg_integer">>, #{example => 100, minimum => 1, type => integer}},
         {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}},
@@ -176,7 +184,8 @@ t_complicated_type(_Config) ->
         {<<"comma_separated_list">>, #{example => <<"item1,item2">>, type => string}},
         {<<"comma_separated_atoms">>, #{example => <<"item1,item2">>, type => string}},
         {<<"log_level">>,
-            #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all], type => string}},
+            #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all],
+                type => string}},
         {<<"fix_integer">>, #{default => 100, enum => [100], example => 100,type => integer}}
     ],
         <<"type">> => object}}}},
@@ -192,15 +201,20 @@ t_ref_array_with_key(_Config) ->
     Path = "/ref/array/with/key",
     Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{
         required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [
-            {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+            {<<"per_page">>, #{description => <<"good per page desc">>,
+                example => 1, maximum => 100, minimum => 1, type => integer}},
             {<<"timeout">>, #{default => 5, <<"oneOf">> =>
             [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
             {<<"assert">>, #{description => <<"money">>, example => 3.14159, type => number}},
-            {<<"number_ex">>, #{description => <<"number example">>, example => 42, type => number}},
+            {<<"number_ex">>, #{description => <<"number example">>,
-            {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>, type => number}},
+                example => 42, type => number}},
-            {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>, type => string}},
+            {<<"percent_ex">>, #{description => <<"percent example">>,
+                example => <<"12%">>, type => number}},
+            {<<"duration_ms_ex">>, #{description => <<"duration ms example">>,
+                example => <<"32s">>, type => string}},
             {<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}},
-            {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}}
+            {<<"array_refs">>, #{items => #{<<"$ref">> =>
+            <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}}
         ]}
     }}},
     ExpectRefs = [{?MODULE, good_ref}],
@@ -227,25 +241,34 @@ t_hocon_schema_function(_Config) ->
         }},
         #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"type">> => object,
             <<"properties">> => [
-                {<<"page">>, #{description => <<"good page">>, example => 1, maximum => 100, minimum => 1, type => integer}},
+                {<<"page">>, #{description => <<"good page">>,
-                {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}
+                    example => 1, maximum => 100, minimum => 1, type => integer}},
+                {<<"another_ref">>, #{<<"$ref">> =>
+                <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}
             ]
         }},
         #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object,
             <<"properties">> => [
-                {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>, type => string}},
+                {<<"ip">>, #{description => <<"IP:Port">>,
-                {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}]
+                    example => <<"127.0.0.1:80">>, type => string}},
+                {<<"version">>, #{description => <<"a good version">>,
+                    example => <<"1.0.0">>, type => string}}]
         }},
-        #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>],
+        #{<<"emqx_swagger_remote_schema.root">> =>
+        #{required => [<<"default_password">>, <<"default_username">>],
             <<"properties">> => [{<<"listeners">>, #{items =>
             #{<<"oneOf">> =>
             [#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>},
-                #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, type => array}},
+                #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]},
+                type => array}},
                 {<<"default_username">>,
                     #{default => <<"admin">>, example => <<"string-example">>, type => string}},
-                {<<"default_password">>, #{default => <<"public">>, example => <<"string-example">>, type => string}},
+                {<<"default_password">>,
-                {<<"sample_interval">>, #{default => <<"10s">>, example => <<"1h">>, type => string}},
+                    #{default => <<"public">>, example => <<"string-example">>, type => string}},
-                {<<"token_expired_time">>, #{default => <<"30m">>, example => <<"12m">>, type => string}}],
+                {<<"sample_interval">>,
+                    #{default => <<"10s">>, example => <<"1h">>, type => string}},
+                {<<"token_expired_time">>,
+                    #{default => <<"30m">>, example => <<"12m">>, type => string}}],
             <<"type">> => object}}],
     ExpectRefs = [{emqx_swagger_remote_schema, "root"}],
     {_, Components} = validate(Path, Object, ExpectRefs),

apps/emqx_gateway/src/coap/emqx_coap_impl.erl

@@ -89,8 +89,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n",
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                  [GwName, Type, LisName, ListenOnStr]),
+                          [GwName, Type, LisName, ListenOnStr]),
             Pid;
         {error, Reason} ->
             ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
@@ -118,8 +118,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+        ok ->
-                    [GwName, Type, LisName, ListenOnStr]);
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
         {error, Reason} ->
             ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
                   [GwName, Type, LisName, ListenOnStr, Reason])
@@ -129,3 +130,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
 stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
     Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
     esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.

apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl

@@ -59,12 +59,18 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) ->
                   SslOpts ->
                       [{ssl_options, SslOpts}]
               end,
-    _ = grpc:start_server(GwName, ListenOn, Services, SvrOptions),
+    case grpc:start_server(GwName, ListenOn, Services, SvrOptions) of
-    ?ULOG("Start ~ts gRPC server on ~p successfully.~n", [GwName, ListenOn]).
+        {ok, _SvrPid} ->
+            console_print("Start ~ts gRPC server on ~p successfully.",
+                          [GwName, ListenOn]);
+        {error, Reason} ->
+            ?ELOG("Falied to start ~ts gRPC server on ~p, reason: ~p",
+                  [GwName, ListenOn, Reason])
+    end.
 
 stop_grpc_server(GwName) ->
     _ = grpc:stop_server(GwName),
-    ?ULOG("Stop ~s gRPC server successfully.~n", [GwName]).
+    console_print("Stop ~s gRPC server successfully.~n", [GwName]).
 
 start_grpc_client_channel(_GwName, undefined) ->
     undefined;
@@ -157,8 +163,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n",
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                  [GwName, Type, LisName, ListenOnStr]),
+                          [GwName, Type, LisName, ListenOnStr]),
             Pid;
         {error, Reason} ->
             ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
@@ -212,8 +218,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+        ok ->
-                    [GwName, Type, LisName, ListenOnStr]);
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
         {error, Reason} ->
             ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
                   [GwName, Type, LisName, ListenOnStr, Reason])
@@ -223,3 +230,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
 stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
     Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
     esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.

apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl

@@ -91,8 +91,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n",
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                  [GwName, Type, LisName, ListenOnStr]),
+                          [GwName, Type, LisName, ListenOnStr]),
             Pid;
         {error, Reason} ->
             ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
@@ -131,8 +131,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+        ok ->
-                    [GwName, Type, LisName, ListenOnStr]);
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
         {error, Reason} ->
             ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
                   [GwName, Type, LisName, ListenOnStr, Reason])
@@ -142,3 +143,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
 stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
     Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
     esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.

apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl

@@ -108,8 +108,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n",
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                  [GwName, Type, LisName, ListenOnStr]),
+                          [GwName, Type, LisName, ListenOnStr]),
             Pid;
         {error, Reason} ->
             ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
@@ -142,8 +142,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+        ok ->
-                    [GwName, Type, LisName, ListenOnStr]);
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
         {error, Reason} ->
             ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
                   [GwName, Type, LisName, ListenOnStr, Reason])
@@ -153,3 +154,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
 stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
     Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
     esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.

apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl

@@ -93,8 +93,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n",
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                  [GwName, Type, LisName, ListenOnStr]),
+                          [GwName, Type, LisName, ListenOnStr]),
             Pid;
         {error, Reason} ->
             ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
@@ -127,8 +127,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+        ok ->
-                    [GwName, Type, LisName, ListenOnStr]);
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
         {error, Reason} ->
             ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
                   [GwName, Type, LisName, ListenOnStr, Reason])
@@ -138,3 +139,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
 stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
     Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
     esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.

apps/emqx_gateway/test/emqx_exproto_echo_svr.erl

@@ -40,7 +40,7 @@
         , on_received_messages/2
         ]).
 
--define(LOG(Fmt, Args), io:format(standard_error, Fmt, Args)).
+-define(LOG(Fmt, Args), ct:pal(Fmt, Args)).
 
 -define(HTTP, #{grpc_opts => #{service_protos => [emqx_exproto_pb],
                                services => #{'emqx.exproto.v1.ConnectionHandler' => ?MODULE}},
@@ -188,13 +188,15 @@ on_received_messages(Stream, _Md) ->
 %%--------------------------------------------------------------------
 
 handle_in(Conn, ?TYPE_CONNECT, #{<<"clientinfo">> := ClientInfo, <<"password">> := Password}) ->
-    NClientInfo = maps:from_list([{binary_to_atom(K, utf8), V} || {K, V} <- maps:to_list(ClientInfo)]),
+    NClientInfo = maps:from_list(
+                    [{binary_to_atom(K, utf8), V}
+                     || {K, V} <- maps:to_list(ClientInfo)]),
     case ?authenticate(#{conn => Conn, clientinfo => NClientInfo, password => Password}) of
         {ok, #{code := 'SUCCESS'}, _} ->
             case maps:get(keepalive, NClientInfo, 0) of
                 0 -> ok;
                 Intv ->
-                    io:format("Try call start_timer with ~ps", [Intv]),
+                    ?LOG("Try call start_timer with ~ps", [Intv]),
                     ?start_timer(#{conn => Conn, type => 'KEEPALIVE', interval => Intv})
             end,
             handle_out(Conn, ?TYPE_CONNACK, 0);

apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl

@@ -36,7 +36,7 @@
 -define(FLAG_RETAIN(X),X).
 -define(FLAG_SESSION(X),X).
 
--define(LOG(Format, Args), ct:print("TEST: " ++ Format, Args)).
+-define(LOG(Format, Args), ct:pal("TEST: " ++ Format, Args)).
 
 -define(MAX_PRED_TOPIC_ID, 2).
 -define(PREDEF_TOPIC_ID1, 1).
@@ -51,6 +51,8 @@
 -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-",
                                     integer_to_list(erlang:system_time())])).
 
+-elvis([{elvis_style, dont_repeat_yourself, disable}]).
+
 -define(CONF_DEFAULT, <<"
 gateway.mqttsn {
   gateway_id = 1
@@ -177,7 +179,7 @@ t_subscribe_case02(_) ->
     Will = 0,
     CleanSession = 0,
     MsgId = 1,
-    TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1
+    TopicId = ?PREDEF_TOPIC_ID1,
     ReturnCode = 0,
     {ok, Socket} = gen_udp:open(0, [binary]),
 
@@ -227,8 +229,14 @@ t_subscribe_case03(_) ->
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
     gen_udp:close(Socket).
 
-%%In this case We use predefined topic name to register and subcribe, and expect to receive the corresponding predefined topic id but not a new generated topic id from broker. We design this case to illustrate
+%% In this case We use predefined topic name to register and subcribe,
-%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. Once we give more restrictions to different topic id type, this case would be deleted or modified.
+%% and expect to receive the corresponding predefined topic id but not a new
+%% generated topic id from broker. We design this case to illustrate
+%% emqx_sn_gateway's compatibility of dealing with predefined and normal
+%% topics.
+%%
+%% Once we give more restrictions to different topic id type, this case
+%% would be deleted or modified.
 t_subscribe_case04(_) ->
     Dup = 0,
     QoS = 0,
@@ -236,7 +244,7 @@ t_subscribe_case04(_) ->
     Will = 0,
     CleanSession = 0,
     MsgId = 1,
-    TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1
+    TopicId = ?PREDEF_TOPIC_ID1,
     ReturnCode = 0,
     {ok, Socket} = gen_udp:open(0, [binary]),
     ClientId = ?CLIENTID,
@@ -246,8 +254,12 @@ t_subscribe_case04(_) ->
     send_register_msg(Socket, Topic1, MsgId),
     ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)),
     send_subscribe_msg_normal_topic(Socket, QoS, Topic1, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
 
     send_unsubscribe_msg_normal_topic(Socket, Topic1, MsgId),
     ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)),
@@ -277,21 +289,36 @@ t_subscribe_case05(_) ->
     ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"abcD">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId1:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"/sport/#">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"/a/+/water">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"/Tom/Home">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1,
+    ?assertEqual(
-                   ?SN_NORMAL_TOPIC:2, TopicId2:16, MsgId:16, ReturnCode>>,
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
-                 receive_response(Socket)),
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId2:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
     send_unsubscribe_msg_normal_topic(Socket, <<"abcD">>, MsgId),
     ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)),
 
@@ -316,19 +343,37 @@ t_subscribe_case06(_) ->
 
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
     send_register_msg(Socket, <<"abc">>, MsgId),
-    ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)),
+    ?assertEqual(
+       <<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>,
+       receive_response(Socket)
+      ),
 
     send_register_msg(Socket, <<"/blue/#">>, MsgId),
-    ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)),
+    ?assertEqual(
+       <<7, ?SN_REGACK, TopicId0:16,
+         MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>,
+       receive_response(Socket)
+      ),
 
     send_register_msg(Socket, <<"/blue/+/white">>, MsgId),
-    ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)),
+    ?assertEqual(
+       <<7, ?SN_REGACK, TopicId0:16,
+         MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>,
+       receive_response(Socket)
+      ),
     send_register_msg(Socket, <<"/$sys/rain">>, MsgId),
-    ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, receive_response(Socket)),
+    ?assertEqual(
+       <<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>,
+       receive_response(Socket)
+      ),
 
     send_subscribe_msg_short_topic(Socket, QoS, <<"Q2">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1,
+         CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ReturnCode>>,
+       receive_response(Socket)
+      ),
 
     send_unsubscribe_msg_normal_topic(Socket, <<"Q2">>, MsgId),
     ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)),
@@ -352,8 +397,12 @@ t_subscribe_case07(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>,
+       receive_response(Socket)
+      ),
 
     send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId),
     ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)),
@@ -375,8 +424,12 @@ t_subscribe_case08(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>,
+       receive_response(Socket)
+      ),
 
     send_disconnect_msg(Socket, undefined),
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
@@ -400,15 +453,21 @@ t_publish_negqos_case09(_) ->
 
 
     send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-                 receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
     MsgId1 = 3,
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_normal_topic(Socket, NegQoS, MsgId1, TopicId1, Payload1),
     timer:sleep(100),
     case ?ENABLE_QOS3 of
         true  ->
-            Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+            Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+                     Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+                     TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
             What = receive_response(Socket),
             ?assertEqual(Eexp, What)
     end,
@@ -441,7 +500,9 @@ t_publish_qos0_case01(_) ->
     send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1),
     timer:sleep(100),
 
-    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+             TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
     What = receive_response(Socket),
     ?assertEqual(Eexp, What),
 
@@ -463,15 +524,21 @@ t_publish_qos0_case02(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     MsgId1 = 3,
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_predefined_topic(Socket, QoS, MsgId1, PredefTopicId, Payload1),
     timer:sleep(100),
 
-    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2,
+             PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
     What = receive_response(Socket),
     ?assertEqual(Eexp, What),
 
@@ -494,15 +561,21 @@ t_publish_qos0_case3(_) ->
 
     Topic = <<"/a/b/c">>,
     send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     MsgId1 = 3,
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_predefined_topic(Socket, QoS, MsgId1, TopicId, Payload1),
     timer:sleep(100),
 
-    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+             TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
     What = receive_response(Socket),
     ?assertEqual(Eexp, What),
 
@@ -524,8 +597,12 @@ t_publish_qos0_case04(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"#">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     MsgId1 = 2,
     Payload1 = <<20, 21, 22, 23>>,
@@ -533,7 +610,9 @@ t_publish_qos0_case04(_) ->
     send_publish_msg_short_topic(Socket, QoS, MsgId1, Topic, Payload1),
     timer:sleep(100),
 
-    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2,
+             Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
     What = receive_response(Socket),
     ?assertEqual(Eexp, What),
 
@@ -554,8 +633,12 @@ t_publish_qos0_case05(_) ->
     send_connect_msg(Socket, ClientId),
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
     send_subscribe_msg_short_topic(Socket, QoS, <<"/#">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
     send_disconnect_msg(Socket, undefined),
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
 
@@ -577,15 +660,21 @@ t_publish_qos0_case06(_) ->
 
     Topic = <<"abc">>,
     send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     MsgId1 = 3,
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1),
     timer:sleep(100),
 
-    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
+    Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+             TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>,
     What = receive_response(Socket),
     ?assertEqual(Eexp, What),
 
@@ -607,16 +696,24 @@ t_publish_qos1_case01(_) ->
     send_connect_msg(Socket, ClientId),
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
     send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1,
+    ?assertEqual(
-                   ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
-                 receive_response(Socket)),
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1),
-    ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16,
+                   MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)),
     timer:sleep(100),
 
-    ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)),
+    ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+                   Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+                   TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>,
+                 receive_response(Socket)
+                ),
 
     send_disconnect_msg(Socket, undefined),
     gen_udp:close(Socket).
@@ -635,12 +732,17 @@ t_publish_qos1_case02(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1),
-    ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+                 receive_response(Socket)),
     timer:sleep(100),
 
     send_disconnect_msg(Socket, undefined),
@@ -655,7 +757,8 @@ t_publish_qos1_case03(_) ->
     send_connect_msg(Socket, ClientId),
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
     send_publish_msg_predefined_topic(Socket, QoS, MsgId, tid(5), <<20, 21, 22, 23>>),
-    ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>,
+                 receive_response(Socket)),
 
     send_disconnect_msg(Socket, undefined),
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
@@ -674,15 +777,19 @@ t_publish_qos1_case04(_) ->
     send_connect_msg(Socket, ClientId),
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
     send_subscribe_msg_short_topic(Socket, QoS, <<"ab">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1,
+    ?assertEqual(
-                   ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
-                 receive_response(Socket)),
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     Topic = <<"ab">>,
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_short_topic(Socket, QoS, MsgId, Topic, Payload1),
     <<TopicIdShort:16>> = Topic,
-    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+                 receive_response(Socket)),
     timer:sleep(100),
 
     send_disconnect_msg(Socket, undefined),
@@ -708,7 +815,9 @@ t_publish_qos1_case05(_) ->
 
     send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/#">>, <<20, 21, 22, 23>>),
     <<TopicIdShort:16>> = <<"/#">>,
-    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16,
+                   MsgId:16, ?SN_RC_NOT_SUPPORTED>>,
+                 receive_response(Socket)),
 
     send_disconnect_msg(Socket, undefined),
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
@@ -734,7 +843,8 @@ t_publish_qos1_case06(_) ->
 
     send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/+">>, <<20, 21, 22, 23>>),
     <<TopicIdShort:16>> = <<"/+">>,
-    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)),
+    ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16,
+                   MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)),
 
     send_disconnect_msg(Socket, undefined),
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
@@ -761,7 +871,12 @@ t_publish_qos2_case01(_) ->
     send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1),
     ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)),
     send_pubrel_msg(Socket, MsgId),
-    ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)),
+    ?assertEqual(
+       <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+         TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>,
+       receive_response(Socket)
+      ),
     ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)),
     timer:sleep(100),
 
@@ -783,15 +898,23 @@ t_publish_qos2_case02(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, ?FNU:1, QoS:2,
+         ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1),
     ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)),
     send_pubrel_msg(Socket, MsgId),
 
-    ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC :2, PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)),
+    ?assertEqual(
+       <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2,
+         PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>,
+       receive_response(Socket)
+      ),
     ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)),
 
     timer:sleep(100),
@@ -814,15 +937,23 @@ t_publish_qos2_case03(_) ->
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 
     send_subscribe_msg_normal_topic(Socket, QoS, <<"/#">>, MsgId),
-    ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+    ?assertEqual(
-        receive_response(Socket)),
+       <<8, ?SN_SUBACK, ?FNU:1, QoS:2,
+         ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+       receive_response(Socket)
+      ),
 
     Payload1 = <<20, 21, 22, 23>>,
     send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1),
     ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)),
     send_pubrel_msg(Socket, MsgId),
 
-    ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)),
+    ?assertEqual(
+       <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+         Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2,
+         <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>,
+       receive_response(Socket)
+      ),
     ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)),
     timer:sleep(100),
 
@@ -1095,7 +1226,12 @@ t_will_case06(_) ->
 %    send_register_msg(Socket, TopicName1, MsgId1),
 %    ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)),
 %    send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId),
-%    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)),
+%    ?assertEqual(
+%       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+%         WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+%         TopicId1:16, MsgId:16, ReturnCode>>,
+%       receive_response(Socket)
+%      ),
 %
 %    % goto asleep state
 %    send_disconnect_msg(Socket, 1),
@@ -1121,7 +1257,9 @@ t_will_case06(_) ->
 %
 %    %% the broker should sent dl msgs to the awake client before sending the pingresp
 %    UdpData = receive_response(Socket),
-%    MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData),
+%    MsgId_udp = check_publish_msg_on_udp(
+%                  {Dup, QoS, Retain, WillBit, CleanSession,
+%                   ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData),
 %    send_puback_msg(Socket, TopicId1, MsgId_udp),
 %
 %    %% check the pingresp is received at last
@@ -1153,8 +1291,12 @@ t_will_case06(_) ->
 %    CleanSession = 0,
 %    ReturnCode = 0,
 %    send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1),
-%    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>,
+%    ?assertEqual(
-%        receive_response(Socket)),
+%       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+%         WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+%         TopicId0:16, MsgId1:16, ReturnCode>>,
+%       receive_response(Socket)
+%      ),
 %
 %    % goto asleep state
 %    send_disconnect_msg(Socket, 1),
@@ -1188,11 +1330,15 @@ t_will_case06(_) ->
 %    send_regack_msg(Socket, TopicIdNew, MsgId3),
 %
 %    UdpData2 = receive_response(Socket),
-%    MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2),
+%    MsgId_udp2 = check_publish_msg_on_udp(
+%                   {Dup, QoS, Retain, WillBit, CleanSession,
+%                    ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2),
 %    send_puback_msg(Socket, TopicIdNew, MsgId_udp2),
 %
 %    UdpData3 = receive_response(Socket),
-%    MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3),
+%    MsgId_udp3 = check_publish_msg_on_udp(
+%                   {Dup, QoS, Retain, WillBit, CleanSession,
+%                    ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3),
 %    send_puback_msg(Socket, TopicIdNew, MsgId_udp3),
 %
 %    ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)),
@@ -1228,8 +1374,12 @@ receive_publish(Socket) ->
 %    CleanSession = 0,
 %    ReturnCode = 0,
 %    send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1),
-%    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>,
+%    ?assertEqual(
-%        receive_response(Socket)),
+%       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+%         WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+%         TopicId0:16, MsgId1:16, ReturnCode>>,
+%       receive_response(Socket)
+%      ),
 %
 %    % goto asleep state
 %    SleepDuration = 30,
@@ -1262,12 +1412,16 @@ receive_publish(Socket) ->
 %    send_regack_msg(Socket, TopicIdNew, MsgId_reg),
 %
 %    UdpData2 = receive_response(Socket),
-%    MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2),
+%    MsgId2 = check_publish_msg_on_udp(
+%               {Dup, QoS, Retain, WillBit, CleanSession,
+%                ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2),
 %    send_puback_msg(Socket, TopicIdNew, MsgId2),
 %    timer:sleep(50),
 %
 %    UdpData3 = wrap_receive_response(Socket),
-%    MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3),
+%    MsgId3 = check_publish_msg_on_udp(
+%               {Dup, QoS, Retain, WillBit, CleanSession,
+%                ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3),
 %    send_puback_msg(Socket, TopicIdNew, MsgId3),
 %    timer:sleep(50),
 %
@@ -1334,7 +1488,9 @@ receive_publish(Socket) ->
 %    send_pingreq_msg(Socket, ClientId),
 %
 %    UdpData = wrap_receive_response(Socket),
-%    MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData),
+%    MsgId_udp = check_publish_msg_on_udp(
+%                  {Dup, QoS, Retain, WillBit, CleanSession,
+%                   ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData),
 %    send_pubrec_msg(Socket, MsgId_udp),
 %    ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)),
 %    send_pubcomp_msg(Socket, MsgId_udp),
@@ -1369,8 +1525,12 @@ receive_publish(Socket) ->
 %    send_register_msg(Socket, TopicName_tom, MsgId1),
 %    TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)),
 %    send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1),
-%    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>,
+%    ?assertEqual(
-%        receive_response(Socket)),
+%       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+%         WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+%         TopicId_tom:16, MsgId1:16, ReturnCode>>,
+%       receive_response(Socket)
+%      ),
 %
 %    % goto asleep state
 %    send_disconnect_msg(Socket, SleepDuration),
@@ -1448,8 +1608,12 @@ receive_publish(Socket) ->
 %    CleanSession = 0,
 %    ReturnCode = 0,
 %    send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1),
-%    ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>,
+%    ?assertEqual(
-%    receive_response(Socket)),
+%       <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1,
+%         WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+%         TopicId0:16, MsgId1:16, ReturnCode>>,
+%       receive_response(Socket)
+%      ),
 %    % goto asleep state
 %    SleepDuration = 30,
 %    send_disconnect_msg(Socket, SleepDuration),
@@ -1483,7 +1647,9 @@ receive_publish(Socket) ->
 %        udp_receive_timeout ->
 %            ok;
 %        UdpData2 ->
-%            MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2),
+%            MsgId2 = check_publish_msg_on_udp(
+%                       {Dup, QoS, Retain, WillBit, CleanSession,
+%                        ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2),
 %            send_puback_msg(Socket, TopicIdNew, MsgId2)
 %    end,
 %    timer:sleep(100),
@@ -1492,7 +1658,9 @@ receive_publish(Socket) ->
 %        udp_receive_timeout ->
 %            ok;
 %        UdpData3 ->
-%            MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3),
+%            MsgId3 = check_publish_msg_on_udp(
+%                       {Dup, QoS, Retain, WillBit, CleanSession,
+%                        ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3),
 %            send_puback_msg(Socket, TopicIdNew, MsgId3)
 %    end,
 %    timer:sleep(100),
@@ -1510,7 +1678,8 @@ receive_publish(Socket) ->
 %
 %    %% send PINGREQ again to enter awake state
 %    send_pingreq_msg(Socket, ClientId),
-%    %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here
+%    %% will not receive any buffered PUBLISH messages buffered before last
+%    %% awake, only receive PINGRESP here
 %    ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)),
 %
 %    gen_udp:close(Socket).
@@ -1913,8 +2082,11 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket
     PubMsg = receive_response(Socket),
     Length = 7 + byte_size(Payload),
     ?LOG("check_dispatched_message ~p~n", [PubMsg]),
-    ?LOG("expected ~p xx ~p~n", [<<Length, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, ?FNU:2, TopicIdType:2, TopicId:16>>, Payload]),
+    ?LOG("expected ~p xx ~p~n",
-    <<Length, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, ?FNU:2, TopicIdType:2, TopicId:16, MsgId:16, Payload/binary>> = PubMsg,
+         [<<Length, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+            ?FNU:2, TopicIdType:2, TopicId:16>>, Payload]),
+    <<Length, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+      ?FNU:2, TopicIdType:2, TopicId:16, MsgId:16, Payload/binary>> = PubMsg,
     case QoS of
         0 -> ok;
         1 -> send_puback_msg(Socket, TopicId, MsgId);
@@ -1926,11 +2098,13 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket
 get_udp_broadcast_address() ->
     "255.255.255.255".
 
-check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, TopicType, TopicId, Payload}, UdpData) ->
+check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession,
+                          TopicType, TopicId, Payload}, UdpData) ->
     <<HeaderUdp:5/binary, MsgId:16, PayloadIn/binary>> = UdpData,
     ct:pal("UdpData: ~p, Payload: ~p, PayloadIn: ~p", [UdpData, Payload, PayloadIn]),
     Size9 = byte_size(Payload) + 7,
-    Eexp = <<Size9:8, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, TopicType:2, TopicId:16>>,
+    Eexp = <<Size9:8, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1,
+             WillBit:1, CleanSession:1, TopicType:2, TopicId:16>>,
     ?assertEqual(Eexp, HeaderUdp),     % mqtt-sn header should be same
     ?assertEqual(Payload, PayloadIn),  % payload should be same
     MsgId.

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -148,7 +148,6 @@ init_per_testcase(t_events, Config) ->
         #{id => <<"rule:t_events">>,
             sql => SQL,
             outputs => [
-                #{function => console},
                 #{function => <<"emqx_rule_engine_SUITE:output_record_triggered_events">>,
                   args => #{}}
             ],

bin/emqx

@@ -587,6 +587,7 @@ case "${COMMAND}" in
 
     ping)
         assert_node_alive
+        echo pong
         ;;
 
     escript)

build

@@ -52,13 +52,10 @@ log() {
 }
 
 docgen() {
-    local conf_doc_html libs_dir1 libs_dir2
+    local libs_dir1 libs_dir2
-    conf_doc_html="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.html"
-    echo "===< Generating config document $conf_doc_html"
     libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
     libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
-    # shellcheck disable=SC2086
+
-    erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_html', hocon_schema_html:gen(emqx_conf_schema, \"EMQ X ${PKG_VSN}\")), halt(0)."
     local conf_doc_markdown
     conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md"
     echo "===< Generating config document $conf_doc_markdown"
@@ -146,6 +143,7 @@ make_zip() {
     ## for DEB and RPM packages the dependencies are resoved by yum and apt
     cp_dyn_libs "${tard}/emqx"
     (cd "${tard}" && zip -qr - emqx) > "${zipball}"
+    log "Zip package successfully created: ${zipball}"
 }
 
 ## This function builds the default docker image based on alpine:3.14 (by default)

deploy/docker/Dockerfile

@@ -28,6 +28,7 @@ RUN cd /emqx \
 
 FROM $RUN_FROM
 
+## define ARG again after 'FROM $RUN_FROM'
 ARG EMQX_NAME=emqx
 
 COPY deploy/docker/docker-entrypoint.sh /usr/bin/

rebar.config.erl

@@ -148,44 +148,52 @@ prod_overrides() ->
 
 profiles() ->
     Vsn = get_vsn(),
-    [ {'emqx',          [ {erl_opts, prod_compile_opts()}
+    [ {'emqx',
-                        , {relx, relx(Vsn, cloud, bin, ce)}
+       [ {erl_opts, prod_compile_opts()}
-                        , {overrides, prod_overrides()}
+       , {relx, relx(Vsn, cloud, bin, ce)}
-                        , {project_app_dirs, project_app_dirs(ce)}
+       , {overrides, prod_overrides()}
-                        ]}
+       , {project_app_dirs, project_app_dirs(ce)}
-    , {'emqx-pkg',      [ {erl_opts, prod_compile_opts()}
+       ]}
-                        , {relx, relx(Vsn, cloud, pkg, ce)}
+    , {'emqx-pkg',
-                        , {overrides, prod_overrides()}
+       [ {erl_opts, prod_compile_opts()}
-                        , {project_app_dirs, project_app_dirs(ce)}
+       , {relx, relx(Vsn, cloud, pkg, ce)}
-                        ]}
+       , {overrides, prod_overrides()}
-    , {'emqx-ee',       [ {erl_opts, prod_compile_opts()}
+       , {project_app_dirs, project_app_dirs(ce)}
-                        , {relx, relx(Vsn, cloud, bin, ee)}
+       ]}
-                        , {overrides, prod_overrides()}
+    , {'emqx-enterprise',
-                        , {project_app_dirs, project_app_dirs(ee)}
+       [ {erl_opts, prod_compile_opts()}
-                        ]}
+       , {relx, relx(Vsn, cloud, bin, ee)}
-    , {'emqx-ee-pkg',   [ {erl_opts, prod_compile_opts()}
+       , {overrides, prod_overrides()}
-                        , {relx, relx(Vsn, cloud, pkg, ee)}
+       , {project_app_dirs, project_app_dirs(ee)}
-                        , {overrides, prod_overrides()}
+       ]}
-                        , {project_app_dirs, project_app_dirs(ee)}
+    , {'emqx-enterprise-pkg',
-                        ]}
+       [ {erl_opts, prod_compile_opts()}
-    , {'emqx-edge',     [ {erl_opts, prod_compile_opts()}
+       , {relx, relx(Vsn, cloud, pkg, ee)}
-                        , {relx, relx(Vsn, edge, bin, ce)}
+       , {overrides, prod_overrides()}
-                        , {overrides, prod_overrides()}
+       , {project_app_dirs, project_app_dirs(ee)}
-                        , {project_app_dirs, project_app_dirs(ce)}
+       ]}
-                        ]}
+    , {'emqx-edge',
-    , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()}
+       [ {erl_opts, prod_compile_opts()}
-                        , {relx, relx(Vsn, edge, pkg, ce)}
+       , {relx, relx(Vsn, edge, bin, ce)}
-                        , {overrides, prod_overrides()}
+       , {overrides, prod_overrides()}
-                        , {project_app_dirs, project_app_dirs(ce)}
+       , {project_app_dirs, project_app_dirs(ce)}
-                        ]}
+       ]}
-    , {check,           [ {erl_opts, common_compile_opts()}
+    , {'emqx-edge-pkg',
-                        , {project_app_dirs, project_app_dirs(ce)}
+       [ {erl_opts, prod_compile_opts()}
-                        ]}
+       , {relx, relx(Vsn, edge, pkg, ce)}
-    , {test,            [ {deps, test_deps()}
+       , {overrides, prod_overrides()}
-                        , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) }
+       , {project_app_dirs, project_app_dirs(ce)}
-                        , {extra_src_dirs, [{"test", [{recursive, true}]}]}
+       ]}
-                        , {project_app_dirs, project_app_dirs(ce)}
+    , {check,
-                        ]}
+       [ {erl_opts, common_compile_opts()}
+       , {project_app_dirs, project_app_dirs(ce)}
+       ]}
+    , {test,
+       [ {deps, test_deps()}
+       , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) }
+       , {extra_src_dirs, [{"test", [{recursive, true}]}]}
+       , {project_app_dirs, project_app_dirs(ce)}
+       ]}
     ].
 
 %% RelType: cloud (full size) | edge (slim size)

scripts/update_appup.escript

@@ -154,7 +154,7 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
     Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
     Filename = filename:join(BaseDir, Dir),
     Script = "mkdir -p ${OUTFILE} &&
-              wget -O ${OUTFILE}.zip ${URL} &&
+              wget -c -O ${OUTFILE}.zip ${URL} &&
               unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
     Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
     bash(Script, Env),
@@ -298,12 +298,15 @@ render_appfile(File, Upgrade, Downgrade) ->
     ok = file:write_file(File, IOList).
 
 create_stub(App) ->
-    case locate(src, App, ".app.src") of
+    Ext = ".app.src",
+    case locate(src, App, Ext) of
         {ok, AppSrc} ->
-            AppupFile = filename:basename(AppSrc) ++ ".appup.src",
+            DirName = filename:dirname(AppSrc),
+            AppupFile = filename:basename(AppSrc, Ext) ++ ".appup.src",
             Default = {<<".*">>, []},
-            render_appfile(AppupFile, [Default], [Default]),
+            AppupFileFullpath = filename:join(DirName, AppupFile),
-            AppupFile;
+            render_appfile(AppupFileFullpath, [Default], [Default]),
+            {ok, AppupFileFullpath};
         undefined ->
             false
     end.