From 8d687e50fd103023c94dd1dedc37cbf69dead91a Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 12:02:38 +0200
Subject: [PATCH 01/27] chore: delete EMQX_NAME and EMQX_HOST env variables

---
 bin/emqx                           |  3 ---
 deploy/docker/docker-entrypoint.sh | 40 ++++++++++++++----------------
 2 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index bc1d00e35..e1ab01dac 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -316,12 +316,9 @@ esac
 ## Possible ways to configure emqx node name:
 ## 1. configure node.name in emqx.conf
 ## 2. override with environment variable EMQX_NODE_NAME
-## 3. override with environment variable EMQX_NAME and EMQX_HOST (deprecated)
 ## Node name is either short-name (without '@'), e.g. 'emqx'
 ## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1'
 NAME="${EMQX_NODE_NAME:-}"
-# to be backward compatible
-[ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NAME="${EMQX_NAME}@${EMQX_HOST}"
 if [ -z "$NAME" ]; then
     if [ "$IS_BOOT_COMMAND" = 'no' ]; then
         # for non-boot commands, inspect vm.<time>.args for node name
diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh
index 6af99ac97..d558c15ee 100755
--- a/deploy/docker/docker-entrypoint.sh
+++ b/deploy/docker/docker-entrypoint.sh
@@ -16,30 +16,26 @@ shopt -s nullglob
 
 LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1)
 
-## EMQ Base settings and plugins setting
-# Base settings in /opt/emqx/etc/emqx.conf
-# Plugin settings in /opt/emqx/etc/plugins
-
-if [[ -z "$EMQX_NAME" ]]; then
-    EMQX_NAME="$(hostname)"
-    export EMQX_NAME
-fi
-
-if [[ -z "$EMQX_HOST" ]]; then
-    if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
-        EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
-        EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
-    elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
-        EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
-        EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
-    else
-        EMQX_HOST="$LOCAL_IP"
-    fi
-    export EMQX_HOST
-fi
-
 if [[ -z "$EMQX_NODE_NAME" ]]; then
+
+    if [[ -z "$EMQX_NAME" ]]; then
+        EMQX_NAME="$(hostname)"
+    fi
+
+    if [[ -z "$EMQX_HOST" ]]; then
+        if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+            EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
+            EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
+        elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+            EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
+            EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
+        else
+            EMQX_HOST="$LOCAL_IP"
+        fi
+    fi
     export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST"
+    unset EMQX_NAME
+    unset EMQX_HOST
 fi
 
 # fill tuples on specific file

From df69a9c031b339c97b0ac8fa01a8ee8ee5b57956 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 15:06:28 +0200
Subject: [PATCH 02/27] chore: upgrade to hocon 0.11.0

---
 apps/emqx/rebar.config        | 2 +-
 apps/emqx/src/emqx_config.erl | 2 +-
 rebar.config                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 55bad7471..7b72f12d3 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -15,7 +15,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.5"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index 43a41cdc6..91341204b 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -209,7 +209,7 @@ init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) ->
 check_config(SchemaModule, RawConf) ->
     Opts = #{return_plain => true,
              nullable => true,
-             is_richmap => false
+             format => map
             },
     {AppEnvs, CheckedConf} =
         hocon_schema:map_translate(SchemaModule, RawConf, Opts),
diff --git a/rebar.config b/rebar.config
index baadf8bfa..d8a8b4de2 100644
--- a/rebar.config
+++ b/rebar.config
@@ -60,7 +60,7 @@
     , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
     , {getopt, "1.0.1"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.5"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.3.0"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}}
     ]}.

From 0f5c5db8415ee833f215491f6ee4ad3964408e77 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 18:33:31 +0200
Subject: [PATCH 03/27] refactor(logger): stop using anonymous fun for logging

---
 apps/emqx/include/logger.hrl          | 25 +++++++++++++++++++------
 apps/emqx/src/emqx_logger_jsonfmt.erl |  8 +++++++-
 apps/emqx/src/emqx_logger_textfmt.erl |  1 +
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index 4881a6413..cfdf584f8 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -14,10 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% debug | info | notice | warning | error | critical | alert | emergency
+-ifndef(EMQX_LOGGER_HRL).
+-define(EMQX_LOGGER_HRL, true).
 
 -compile({parse_transform, emqx_logger}).
 
+%% debug | info | notice | warning | error | critical | alert | emergency
 -define(DEBUG(Format), ?LOG(debug, Format, [])).
 -define(DEBUG(Format, Args), ?LOG(debug, Format, Args)).
 
@@ -41,10 +43,21 @@
 
 -define(LOG(Level, Format), ?LOG(Level, Format, [])).
 
--define(LOG(Level, Format, Args),
-        begin
-          (logger:log(Level,#{},#{report_cb => fun(_) -> {'$logger_header'()++(Format), (Args)} end,
-                                  mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
-                                  line => ?LINE}))
+-define(LOG(Level, Format, Args, Meta),
+        %% check 'allow' here so we do not have to pass an anonymous function
+        %% down to logger which may cause `badfun` exception during upgrade
+        case logger:allow(Level, ?MODULE) of
+            true ->
+                logger:log(Level, (Format), (Args),
+                           (Meta)#{ mfa => <<(atom_to_binary(?MODULE, utf8))/binary, $:,
+                                             (atom_to_binary(?FUNCTION_NAME, utf8))/binary, $/,
+                                             (integer_to_binary(?FUNCTION_ARITY))/binary>>
+                                  , line => ?LINE
+                                  });
+            false ->
+                ok
         end).
 
+-define(LOG(Level, Format, Args), ?LOG(Level, Format, Args, #{})).
+
+-endif.
diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl
index fdc1a8bb3..5cabb6ddb 100644
--- a/apps/emqx/src/emqx_logger_jsonfmt.erl
+++ b/apps/emqx/src/emqx_logger_jsonfmt.erl
@@ -197,12 +197,18 @@ json_obj(Data, Config) ->
                       json_kv(K, V, D, Config)
               end, maps:new(), Data).
 
-json_kv(mfa, {M, F, A}, Data, _Config) -> %% emqx/snabbkaffe
+json_kv(mfa, {M, F, A}, Data, _Config) -> %% snabbkaffe
     maps:put(mfa, <<(atom_to_binary(M, utf8))/binary, $:,
                     (atom_to_binary(F, utf8))/binary, $/,
                     (integer_to_binary(A))/binary>>, Data);
 json_kv('$kind', Kind, Data, Config) -> %% snabbkaffe
     maps:put(msg, json(Kind, Config), Data);
+json_kv(gl, _, Data, _Config) ->
+    %% drop gl because it's not interesting
+    Data;
+json_kv(file, _, Data, _Config) ->
+    %% drop 'file' because we have mfa
+    Data;
 json_kv(K0, V, Data, Config) ->
     K = json_key(K0),
     case is_map(V) of
diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl
index 291153d74..eae637e4d 100644
--- a/apps/emqx/src/emqx_logger_textfmt.erl
+++ b/apps/emqx/src/emqx_logger_textfmt.erl
@@ -26,6 +26,7 @@
         , peername  % formatted as a part of templated message
         , clientid  % formatted as a part of templated message
         , gl        % not interesting
+        , file
         ]).
 
 check_config(X) -> logger_formatter:check_config(X).

From e57f99168b8b7eeb536646622605bcb97b981f66 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 18:35:30 +0200
Subject: [PATCH 04/27] refactor(logger): delete log header parse-transform

because now we have MFA in log
---
 apps/emqx/include/logger.hrl         |  2 --
 apps/emqx/src/emqx_logger.erl        | 37 -----------------------
 apps/emqx/test/emqx_logger_SUITE.erl | 44 ----------------------------
 3 files changed, 83 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index cfdf584f8..80799179f 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -17,8 +17,6 @@
 -ifndef(EMQX_LOGGER_HRL).
 -define(EMQX_LOGGER_HRL, true).
 
--compile({parse_transform, emqx_logger}).
-
 %% debug | info | notice | warning | error | critical | alert | emergency
 -define(DEBUG(Format), ?LOG(debug, Format, [])).
 -define(DEBUG(Format, Args), ?LOG(debug, Format, Args)).
diff --git a/apps/emqx/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl
index a733b0d3a..986ba11e0 100644
--- a/apps/emqx/src/emqx_logger.erl
+++ b/apps/emqx/src/emqx_logger.erl
@@ -56,8 +56,6 @@
         , stop_log_handler/1
         ]).
 
--export([parse_transform/2]).
-
 -type(peername_str() :: list()).
 -type(logger_dst() :: file:filename() | console | unknown).
 -type(logger_handler_info() :: #{
@@ -234,13 +232,6 @@ set_log_level(Level) ->
         {error, Error} -> {error, {primary_logger_level, Error}}
     end.
 
-%% @doc The parse transform for prefixing a module-specific logger header to the logs.
-%% The logger header can be specified by "-logger_header(Header)", where Header
-%% must be a string (list).
-%% @end
-parse_transform(AST, _Opts) ->
-    trans(AST, "", []).
-
 %%--------------------------------------------------------------------
 %% Internal Functions
 %%--------------------------------------------------------------------
@@ -307,31 +298,3 @@ list_stopped_handler_config() ->
         undefined -> [];
         ConfList -> maps:values(ConfList)
     end.
-
-%% @doc The following parse-transforms stripped off the module attribute named
-%% `-logger_header(Header)` (if there's one) from the source code, and then
-%% generate a function named '$logger_header'/0, which returns the logger header.
-%% @end
-trans([], LogHeader, ResAST) ->
-    lists:reverse([header_fun(LogHeader) | ResAST]);
-trans([{eof, L} | AST], LogHeader, ResAST) ->
-    lists:reverse([{eof, L}, header_fun(LogHeader) | ResAST]) ++ AST;
-trans([{attribute, _, module, _Mod} = M | AST], Header, ResAST) ->
-    trans(AST, Header, [export_header_fun(), M | ResAST]);
-trans([{attribute, _, logger_header, Header} | AST], _, ResAST) ->
-    io_lib:printable_list(Header) orelse erlang:error({invalid_string, Header}),
-    trans(AST, Header, ResAST);
-trans([F | AST], LogHeader, ResAST) ->
-    trans(AST, LogHeader, [F | ResAST]).
-
-export_header_fun() ->
-    {attribute,erl_anno:new(0),export,[{'$logger_header',0}]}.
-
-header_fun(LogHeader) ->
-    L = erl_anno:new(0),
-    {function,L,'$logger_header',0,
-         [{clause,L,
-             [], [], [{string,L,pad(LogHeader)}]}]}.
-
-pad("") -> "";
-pad(Str) -> Str ++ " ".
diff --git a/apps/emqx/test/emqx_logger_SUITE.erl b/apps/emqx/test/emqx_logger_SUITE.erl
index a087761b8..d09222b6b 100644
--- a/apps/emqx/test/emqx_logger_SUITE.erl
+++ b/apps/emqx/test/emqx_logger_SUITE.erl
@@ -25,19 +25,6 @@
 -define(a, "a").
 -define(SUPPORTED_LEVELS, [emergency, alert, critical, error, warning, notice, info, debug]).
 
--define(PARSE_TRANS_TEST_CODE,
-        "-module(mytest).\n"
-        "-logger_header(\"[MyTest]\").\n"
-        "-export([run/0]).\n"
-        "-compile({parse_transform, logger_header}).\n"
-        "run() -> '$logger_header'().").
-
--define(PARSE_TRANS_TEST_CODE2,
-        "-module(mytest).\n"
-        "-export([run/0]).\n"
-        "-compile({parse_transform, logger_header}).\n"
-        "run() -> '$logger_header'().").
-
 all() -> emqx_ct:all(?MODULE).
 
 init_per_testcase(_TestCase, Config) ->
@@ -132,37 +119,6 @@ t_start_stop_log_handler2(_) ->
     ?assertMatch({error, {not_started, default}},
                  ?LOGGER:stop_log_handler(default)).
 
-t_parse_transform(_) ->
-    {ok, Toks, _EndLine} = erl_scan:string(?PARSE_TRANS_TEST_CODE),
-    FormToks = split_toks_at_dot(Toks),
-    Forms = [case erl_parse:parse_form(Ts) of
-                 {ok, Form} ->
-                     Form;
-                 {error, Reason} ->
-                     erlang:error({parse_form_error, Ts, Reason})
-             end
-             || Ts <- FormToks],
-    %ct:log("=====: ~p", [Forms]),
-    AST1 = emqx_logger:parse_transform(Forms, []),
-    %ct:log("=====: ~p", [AST1]),
-    ?assertNotEqual(false, lists:keyfind('$logger_header', 3, AST1)).
-
-t_parse_transform_empty_header(_) ->
-    {ok, Toks, _EndLine} = erl_scan:string(?PARSE_TRANS_TEST_CODE2),
-    FormToks = split_toks_at_dot(Toks),
-    Forms = [case erl_parse:parse_form(Ts) of
-                 {ok, Form} ->
-                     Form;
-                 {error, Reason} ->
-                     erlang:error({parse_form_error, Ts, Reason})
-             end
-             || Ts <- FormToks],
-    %ct:log("=====: ~p", [Forms]),
-    AST2 = emqx_logger:parse_transform(Forms++[{eof, 15}], []),
-    %ct:log("=====: ~p", [AST2]),
-    ?assertNotEqual(false, lists:keyfind('$logger_header', 3, AST2)).
-
-
 t_set_metadata_peername(_) ->
     ?assertEqual(ok, ?LOGGER:set_metadata_peername("for_test")).
 

From 5d59ac1f023c409ceaa43297d9ed16fc87e73bd3 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 18:52:05 +0200
Subject: [PATCH 05/27] refactor(logging): delete log headers from all modules

---
 apps/emqx/src/emqx.erl                                        | 1 -
 apps/emqx/src/emqx_alarm.erl                                  | 2 --
 apps/emqx/src/emqx_alarm_handler.erl                          | 1 -
 apps/emqx/src/emqx_banned.erl                                 | 1 -
 apps/emqx/src/emqx_broker.erl                                 | 1 -
 apps/emqx/src/emqx_broker_helper.erl                          | 1 -
 apps/emqx/src/emqx_channel.erl                                | 1 -
 apps/emqx/src/emqx_cm.erl                                     | 1 -
 apps/emqx/src/emqx_cm_registry.erl                            | 1 -
 apps/emqx/src/emqx_connection.erl                             | 1 -
 apps/emqx/src/emqx_ctl.erl                                    | 1 -
 apps/emqx/src/emqx_flapping.erl                               | 4 +---
 apps/emqx/src/emqx_hooks.erl                                  | 1 -
 apps/emqx/src/emqx_metrics.erl                                | 1 -
 apps/emqx/src/emqx_os_mon.erl                                 | 1 -
 apps/emqx/src/emqx_plugins.erl                                | 1 -
 apps/emqx/src/emqx_pool.erl                                   | 1 -
 apps/emqx/src/emqx_psk.erl                                    | 1 -
 apps/emqx/src/emqx_router.erl                                 | 1 -
 apps/emqx/src/emqx_router_helper.erl                          | 1 -
 apps/emqx/src/emqx_session.erl                                | 1 -
 apps/emqx/src/emqx_shared_sub.erl                             | 1 -
 apps/emqx/src/emqx_stats.erl                                  | 1 -
 apps/emqx/src/emqx_sys.erl                                    | 1 -
 apps/emqx/src/emqx_sys_mon.erl                                | 1 -
 apps/emqx/src/emqx_tracer.erl                                 | 1 -
 apps/emqx/src/emqx_ws_connection.erl                          | 1 -
 apps/emqx_authz/src/emqx_authz.erl                            | 1 -
 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl            | 1 -
 apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl              | 1 -
 apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl             | 1 -
 .../src/coap/resources/emqx_coap_mqtt_resource.erl            | 1 -
 .../src/coap/resources/emqx_coap_pubsub_resource.erl          | 1 -
 .../src/coap/resources/emqx_coap_pubsub_topics.erl            | 1 -
 apps/emqx_gateway/src/emqx_gateway_app.erl                    | 1 -
 apps/emqx_gateway/src/emqx_gateway_cm.erl                     | 1 -
 apps/emqx_gateway/src/emqx_gateway_cm_registry.erl            | 1 -
 apps/emqx_gateway/src/emqx_gateway_ctx.erl                    | 1 -
 apps/emqx_gateway/src/emqx_gateway_insta_sup.erl              | 1 -
 apps/emqx_gateway/src/emqx_gateway_metrics.erl                | 1 -
 apps/emqx_gateway/src/emqx_gateway_registry.erl               | 1 -
 apps/emqx_gateway/src/exhook/emqx_exhook.erl                  | 1 -
 apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl          | 1 -
 apps/emqx_gateway/src/exhook/emqx_exhook_server.erl           | 1 -
 apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl        | 1 -
 apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl           | 1 -
 apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl           | 1 -
 apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl              | 1 -
 apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl            | 1 -
 apps/emqx_modules/src/emqx_delayed.erl                        | 1 -
 apps/emqx_modules/src/emqx_topic_metrics.erl                  | 1 -
 apps/emqx_retainer/src/emqx_retainer.erl                      | 1 -
 apps/emqx_retainer/src/emqx_retainer_mnesia.erl               | 1 -
 apps/emqx_rule_engine/src/emqx_rule_engine_api.erl            | 1 -
 apps/emqx_rule_engine/src/emqx_rule_events.erl                | 1 -
 apps/emqx_rule_engine/src/emqx_rule_monitor.erl               | 1 -
 56 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl
index 3530f0dfb..f22b33acc 100644
--- a/apps/emqx/src/emqx.erl
+++ b/apps/emqx/src/emqx.erl
@@ -20,7 +20,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[EMQ X]").
 
 %% Start/Stop the application
 -export([ start/0
diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl
index 223d5aa50..ab3e2d702 100644
--- a/apps/emqx/src/emqx_alarm.erl
+++ b/apps/emqx/src/emqx_alarm.erl
@@ -22,8 +22,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Alarm Handler]").
-
 %% Mnesia bootstrap
 -export([mnesia/1]).
 
diff --git a/apps/emqx/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl
index 2307b79db..06f4e23a6 100644
--- a/apps/emqx/src/emqx_alarm_handler.erl
+++ b/apps/emqx/src/emqx_alarm_handler.erl
@@ -21,7 +21,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Alarm Handler]").
 
 %% gen_event callbacks
 -export([ init/1
diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl
index 16804d329..f0991d967 100644
--- a/apps/emqx/src/emqx_banned.erl
+++ b/apps/emqx/src/emqx_banned.erl
@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Banned]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index eb8023d34..1248f9980 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include("emqx_mqtt.hrl").
 
--logger_header("[Broker]").
 
 -export([start_link/2]).
 
diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl
index 705715c6c..fad4e8713 100644
--- a/apps/emqx/src/emqx_broker_helper.erl
+++ b/apps/emqx/src/emqx_broker_helper.erl
@@ -21,7 +21,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Broker Helper]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 21e2a516c..5e4d11953 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Channel]").
 
 -ifdef(TEST).
 -compile(export_all).
diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index e2c8438a2..f4f5f3981 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -24,7 +24,6 @@
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--logger_header("[CM]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl
index da716ca29..c04f6ccaf 100644
--- a/apps/emqx/src/emqx_cm_registry.erl
+++ b/apps/emqx/src/emqx_cm_registry.erl
@@ -23,7 +23,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Registry]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 21ba5231e..ac66c4daf 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--logger_header("[MQTT]").
 
 -ifdef(TEST).
 -compile(export_all).
diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl
index 84aa54de6..051d4de6f 100644
--- a/apps/emqx/src/emqx_ctl.erl
+++ b/apps/emqx/src/emqx_ctl.erl
@@ -21,7 +21,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[Ctl]").
 
 -export([start_link/0, stop/0]).
 
diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl
index 0b852d88d..c4a523669 100644
--- a/apps/emqx/src/emqx_flapping.erl
+++ b/apps/emqx/src/emqx_flapping.erl
@@ -22,8 +22,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[Flapping]").
-
 -export([start_link/0, stop/0]).
 
 %% API
@@ -162,4 +160,4 @@ start_timer(Zone) ->
 start_timers() ->
     lists:foreach(fun({Zone, _ZoneConf}) ->
             start_timer(Zone)
-        end, maps:to_list(emqx_config:get([zones], #{}))).
\ No newline at end of file
+        end, maps:to_list(emqx_config:get([zones], #{}))).
diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl
index eb2da4276..088bb4085 100644
--- a/apps/emqx/src/emqx_hooks.erl
+++ b/apps/emqx/src/emqx_hooks.erl
@@ -22,7 +22,6 @@
 -include("types.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--logger_header("[Hooks]").
 
 -export([ start_link/0
         , stop/0
diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl
index cd0039791..736bb05b0 100644
--- a/apps/emqx/src/emqx_metrics.erl
+++ b/apps/emqx/src/emqx_metrics.erl
@@ -24,7 +24,6 @@
 -include("emqx_mqtt.hrl").
 -include("emqx.hrl").
 
--logger_header("[Metrics]").
 
 -export([ start_link/0
         , stop/0
diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl
index fdcdf82b3..9fd52c21b 100644
--- a/apps/emqx/src/emqx_os_mon.erl
+++ b/apps/emqx/src/emqx_os_mon.erl
@@ -20,7 +20,6 @@
 
 -include("logger.hrl").
 
--logger_header("[OS_MON]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl
index 3c91e612f..6c99305d4 100644
--- a/apps/emqx/src/emqx_plugins.erl
+++ b/apps/emqx/src/emqx_plugins.erl
@@ -19,7 +19,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Plugins]").
 
 -export([ load/0
         , load/1
diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl
index 0a336264d..8fa950fe3 100644
--- a/apps/emqx/src/emqx_pool.erl
+++ b/apps/emqx/src/emqx_pool.erl
@@ -21,7 +21,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Pool]").
 
 %% APIs
 -export([start_link/2]).
diff --git a/apps/emqx/src/emqx_psk.erl b/apps/emqx/src/emqx_psk.erl
index f8be93056..0c5ca2964 100644
--- a/apps/emqx/src/emqx_psk.erl
+++ b/apps/emqx/src/emqx_psk.erl
@@ -18,7 +18,6 @@
 
 -include("logger.hrl").
 
--logger_header("[PSK]").
 
 %% SSL PSK Callbacks
 -export([lookup/3]).
diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl
index 02ac29cbc..1a5e344f2 100644
--- a/apps/emqx/src/emqx_router.erl
+++ b/apps/emqx/src/emqx_router.erl
@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include_lib("ekka/include/ekka.hrl").
 
--logger_header("[Router]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl
index 8761a250e..5866e86b3 100644
--- a/apps/emqx/src/emqx_router_helper.erl
+++ b/apps/emqx/src/emqx_router_helper.erl
@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Router Helper]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl
index 3e0f56610..f915155cb 100644
--- a/apps/emqx/src/emqx_session.erl
+++ b/apps/emqx/src/emqx_session.erl
@@ -48,7 +48,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Session]").
 
 -ifdef(TEST).
 -compile(export_all).
diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl
index 65c0e4d8d..1968c47d8 100644
--- a/apps/emqx/src/emqx_shared_sub.erl
+++ b/apps/emqx/src/emqx_shared_sub.erl
@@ -23,7 +23,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Shared Sub]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl
index f53549e65..51ba72155 100644
--- a/apps/emqx/src/emqx_stats.erl
+++ b/apps/emqx/src/emqx_stats.erl
@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Stats]").
 
 %% APIs
 -export([ start_link/0
diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl
index 2f3f782e6..e15e7489e 100644
--- a/apps/emqx/src/emqx_sys.erl
+++ b/apps/emqx/src/emqx_sys.erl
@@ -22,7 +22,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[SYS]").
 
 -export([ start_link/0
         , stop/0
diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl
index 54a5c533a..0b981ffec 100644
--- a/apps/emqx/src/emqx_sys_mon.erl
+++ b/apps/emqx/src/emqx_sys_mon.erl
@@ -21,7 +21,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[SYSMON]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx/src/emqx_tracer.erl b/apps/emqx/src/emqx_tracer.erl
index 995712f6c..d05840433 100644
--- a/apps/emqx/src/emqx_tracer.erl
+++ b/apps/emqx/src/emqx_tracer.erl
@@ -19,7 +19,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Tracer]").
 
 %% APIs
 -export([ trace/2
diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl
index 540076eaf..b76567f6c 100644
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[MQTT/WS]").
 
 -ifdef(TEST).
 -compile(export_all).
diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl
index 51600ea81..50c8e5122 100644
--- a/apps/emqx_authz/src/emqx_authz.erl
+++ b/apps/emqx_authz/src/emqx_authz.erl
@@ -20,7 +20,6 @@
 -include("emqx_authz.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[AuthZ]").
 
 -export([ register_metrics/0
         , init/0
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl
index 0075b4a1d..ef4d076a4 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl
@@ -20,7 +20,6 @@
 -include("emqx_bridge_mqtt.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Bridge]").
 
 %% APIs
 -export([ start_link/0
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl
index dfef6973e..630fb4443 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl
@@ -116,7 +116,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
--logger_header("[Bridge]").
 
 %% same as default in-flight limit for emqtt
 -define(DEFAULT_BATCH_SIZE, 32).
diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
index 7349ec310..1f5cff043 100644
--- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
+++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
@@ -20,7 +20,6 @@
 -include_lib("emqx/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[PGW-Conn]").
 
 %% API
 -export([ start_link/3
diff --git a/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl
index f52610492..1fd3d7b8e 100644
--- a/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl
+++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl
@@ -22,7 +22,6 @@
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-RES]").
 
 -export([ init/1
         , stop/1
diff --git a/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl
index c7c13da0c..c750f66dd 100644
--- a/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl
+++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl
@@ -22,7 +22,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-PS-RES]").
 
 -export([ init/1
         , stop/1
diff --git a/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl
index 4ebada566..a0fa15467 100644
--- a/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl
+++ b/apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl
@@ -21,7 +21,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-PS-TOPICS]").
 
 -export([ start_link/0
         , stop/1
diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl
index a25640c9b..582a3d26a 100644
--- a/apps/emqx_gateway/src/emqx_gateway_app.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_app.erl
@@ -20,7 +20,6 @@
 
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Gateway]").
 
 -export([start/2, stop/1]).
 
diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl
index e4d21e1a5..546640a90 100644
--- a/apps/emqx_gateway/src/emqx_gateway_cm.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl
@@ -26,7 +26,6 @@
 -include("include/emqx_gateway.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[PGW-CM]").
 
 %% APIs
 -export([start_link/1]).
diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl
index 4275fdf3e..2c449828e 100644
--- a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl
@@ -19,7 +19,6 @@
 
 -behaviour(gen_server).
 
--logger_header("[PGW-CM-Registy]").
 
 -export([start_link/1]).
 
diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl
index 722d4e549..69baf7593 100644
--- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl
@@ -19,7 +19,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header(["PGW-Ctx"]).
 
 %% @doc The running context for a Connection/Channel process.
 %%
diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
index 7994a6cea..0a3cef556 100644
--- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
@@ -21,7 +21,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Insta-Sup]").
 
 %% APIs
 -export([ start_link/3
diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl
index 461eb3344..d4e39443c 100644
--- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl
@@ -20,7 +20,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Metrics]").
 
 %% APIs
 -export([start_link/1]).
diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl
index dde440756..07cc37bd8 100644
--- a/apps/emqx_gateway/src/emqx_gateway_registry.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl
@@ -19,7 +19,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Registry]").
 
 -behavior(gen_server).
 
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook.erl b/apps/emqx_gateway/src/exhook/emqx_exhook.erl
index b3b3057b6..3d977d2f1 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook.erl
+++ b/apps/emqx_gateway/src/exhook/emqx_exhook.erl
@@ -19,7 +19,6 @@
 -include("src/exhook/include/emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook]").
 
 %% Mgmt APIs
 -export([ enable/2
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl
index 9033fdacc..2bb2dc7f5 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl
+++ b/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl
@@ -20,7 +20,6 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook]").
 
 -export([ on_client_connect/2
         , on_client_connack/3
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl b/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl
index 79bc52b4e..79bb9d32b 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl
+++ b/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl
@@ -19,7 +19,6 @@
 -include("src/exhook/include/emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook Svr]").
 
 -define(CNTER, emqx_exhook_counter).
 -define(PB_CLIENT_MOD, emqx_exhook_v_1_hook_provider_client).
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
index 5301de4e4..38fa27162 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
@@ -21,7 +21,6 @@
 -include_lib("emqx/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto Channel]").
 
 -export([ info/1
         , info/2
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
index 34f0606ef..f30e2b6f7 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
@@ -21,7 +21,6 @@
 
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto gClient]").
 
 %% APIs
 -export([async_call/3]).
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl
index f09a9df7b..346f87452 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl
@@ -22,7 +22,6 @@
 -include("src/exproto/include/emqx_exproto.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto gServer]").
 
 -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)).
 
diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
index d7717fdc4..980fea74f 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
@@ -23,7 +23,6 @@
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[SN-Proto]").
 
 %% API
 -export([ info/1
diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
index ef6e21e66..fa7b2a357 100644
--- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
+++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
@@ -22,7 +22,6 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Stomp-Proto]").
 
 -import(proplists, [get_value/2, get_value/3]).
 
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index 2e0706cbd..c55a8ddab 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -21,7 +21,6 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Delayed]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl
index 7297878fe..5e6a3bc98 100644
--- a/apps/emqx_modules/src/emqx_topic_metrics.erl
+++ b/apps/emqx_modules/src/emqx_topic_metrics.erl
@@ -22,7 +22,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
--logger_header("[TOPIC_METRICS]").
 
 -export([ on_message_publish/1
         , on_message_delivered/2
diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl
index 524078153..86cf98af6 100644
--- a/apps/emqx_retainer/src/emqx_retainer.erl
+++ b/apps/emqx_retainer/src/emqx_retainer.erl
@@ -22,7 +22,6 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Retainer]").
 
 -export([start_link/0]).
 
diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
index 5b6028980..dcfeffa8c 100644
--- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
@@ -23,7 +23,6 @@
 -include_lib("stdlib/include/ms_transform.hrl").
 -include_lib("stdlib/include/qlc.hrl").
 
--logger_header("[Retainer]").
 
 -export([delete_message/2
         , store_retained/2
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 24b4d2c13..122ae7705 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -19,7 +19,6 @@
 -include("rule_engine.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[RuleEngineAPI]").
 
 -rest_api(#{name   => create_rule,
             method => 'POST',
diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl
index 5865f224c..6eacced42 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_events.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl
@@ -20,7 +20,6 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[RuleEvents]").
 
 -export([ load/1
         , unload/0
diff --git a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl
index ed3b23395..dd2f6237c 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl
@@ -20,7 +20,6 @@
 
 -include("rule_engine.hrl").
 -include_lib("emqx/include/logger.hrl").
--logger_header("[Rule Monitor]").
 
 -export([init/1,
          handle_call/3,

From 2998a87a8d07da338559a9e8ba46f8ab902f198d Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 20:42:58 +0200
Subject: [PATCH 06/27] fix(log/jsonfmt): log map() reports at top level JSON
 fields

---
 apps/emqx/src/emqx_logger_jsonfmt.erl | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl
index 5cabb6ddb..5d9094c51 100644
--- a/apps/emqx/src/emqx_logger_jsonfmt.erl
+++ b/apps/emqx/src/emqx_logger_jsonfmt.erl
@@ -42,6 +42,9 @@
 
 -elvis([{elvis_style, no_nested_try_catch, #{ ignore => [emqx_logger_jsonfmt]}}]).
 
+%% this is what used when calling logger:log(Level, Report, Meta).
+-define(DEFAULT_FORMATTER, fun logger:format_otp_report/1).
+
 -type config() :: #{depth       => pos_integer() | unlimited,
                     report_cb   => logger:report_cb(),
                     single_line => boolean()}.
@@ -55,7 +58,11 @@ format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0
 
 format(Msg, Meta, Config) ->
     Data0 =
-        try Meta#{msg => format_msg(Msg, Meta, Config)}
+        try maybe_format_msg(Msg, Meta, Config) of
+            Map when is_map(Map) ->
+                maps:merge(Map, Meta);
+            Bin when is_binary(Bin) ->
+                Meta#{msg => Bin}
         catch
             C:R:S ->
                 Meta#{ msg => "emqx_logger_jsonfmt_format_error"
@@ -68,12 +75,26 @@ format(Msg, Meta, Config) ->
     Data = maps:without([report_cb], Data0),
     jiffy:encode(json_obj(Data, Config)).
 
+maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
+    case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
+        true ->
+            %% reporting a map without a customised format function
+            Report;
+        false ->
+            format_msg(Msg, Meta, Config)
+    end;
+maybe_format_msg(Msg, Meta, Config) ->
+    format_msg(Msg, Meta, Config).
+
 format_msg({string, Chardata}, Meta, Config) ->
+    %% already formatted
     format_msg({"~ts", [Chardata]}, Meta, Config);
 format_msg({report, _} = Msg, Meta, #{report_cb := Fun} = Config)
   when is_function(Fun,1); is_function(Fun,2) ->
+    %% a format callback function in config, no idea when this happens, but leaving it
     format_msg(Msg, Meta#{report_cb => Fun}, maps:remove(report_cb, Config));
 format_msg({report, Report}, #{report_cb := Fun} = Meta, Config) when is_function(Fun, 1) ->
+    %% a format callback function of arity 1
     case Fun(Report) of
         {Format, Args} when is_list(Format), is_list(Args) ->
             format_msg({Format, Args}, maps:remove(report_cb, Meta), Config);
@@ -84,6 +105,7 @@ format_msg({report, Report}, #{report_cb := Fun} = Meta, Config) when is_functio
              }
     end;
 format_msg({report, Report}, #{report_cb := Fun}, Config) when is_function(Fun, 2) ->
+    %% a format callback function of arity 2
     case Fun(Report, maps:with([depth, single_line], Config)) of
         Chardata when ?IS_STRING(Chardata) ->
             try

From 683974e1c855797c0f6aa02a4bf9d9385b8a6c5b Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 21:25:00 +0200
Subject: [PATCH 07/27] refactor(log/textfmt): print mfa metadata in text
 formatter

---
 apps/emqx/include/logger.hrl          |  9 ++++---
 apps/emqx/src/emqx_logger_jsonfmt.erl |  2 +-
 apps/emqx/src/emqx_logger_textfmt.erl | 36 +++++++++++++--------------
 3 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index 80799179f..2c5bea89f 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -47,9 +47,7 @@
         case logger:allow(Level, ?MODULE) of
             true ->
                 logger:log(Level, (Format), (Args),
-                           (Meta)#{ mfa => <<(atom_to_binary(?MODULE, utf8))/binary, $:,
-                                             (atom_to_binary(?FUNCTION_NAME, utf8))/binary, $/,
-                                             (integer_to_binary(?FUNCTION_ARITY))/binary>>
+                           (Meta)#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
                                   , line => ?LINE
                                   });
             false ->
@@ -58,4 +56,9 @@
 
 -define(LOG(Level, Format, Args), ?LOG(Level, Format, Args, #{})).
 
+%% structured logging
+-define(SLOG(Level, Data),
+        logger:log(Level, Data, #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+                                 , line => ?LINE})).
+
 -endif.
diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl
index 5d9094c51..387c9b4c3 100644
--- a/apps/emqx/src/emqx_logger_jsonfmt.erl
+++ b/apps/emqx/src/emqx_logger_jsonfmt.erl
@@ -219,7 +219,7 @@ json_obj(Data, Config) ->
                       json_kv(K, V, D, Config)
               end, maps:new(), Data).
 
-json_kv(mfa, {M, F, A}, Data, _Config) -> %% snabbkaffe
+json_kv(mfa, {M, F, A}, Data, _Config) ->
     maps:put(mfa, <<(atom_to_binary(M, utf8))/binary, $:,
                     (atom_to_binary(F, utf8))/binary, $/,
                     (integer_to_binary(A))/binary>>, Data);
diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl
index eae637e4d..97862a72a 100644
--- a/apps/emqx/src/emqx_logger_textfmt.erl
+++ b/apps/emqx/src/emqx_logger_textfmt.erl
@@ -19,26 +19,24 @@
 -export([format/2]).
 -export([check_config/1]).
 
-%% metadata fields which we do not wish to merge into log data
--define(WITHOUT_MERGE,
-        [ report_cb % just a callback
-        , time      % formatted as a part of templated message
-        , peername  % formatted as a part of templated message
-        , clientid  % formatted as a part of templated message
-        , gl        % not interesting
-        , file
-        ]).
-
 check_config(X) -> logger_formatter:check_config(X).
 
-format(#{msg := Msg0, meta := Meta} = Event, Config) ->
-    Msg = maybe_merge(Msg0, Meta),
-    logger_formatter:format(Event#{msg := Msg}, Config).
+format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) ->
+    logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config);
+format(#{msg := {Fmt, Args}, meta := Meta} = Event, Config) when is_list(Fmt) ->
+    {NewFmt, NewArgs} = enrich_fmt(Fmt, Args, Meta),
+    logger_formatter:format(Event#{msg := {NewFmt, NewArgs}}, Config).
 
-maybe_merge({report, Report}, Meta) when is_map(Report) ->
-    {report, maps:merge(Report, filter(Meta))};
-maybe_merge(Report, _Meta) ->
-    Report.
+enrich(Report, #{mfa := Mfa, line := Line}) ->
+    Report#{mfa => mfa(Mfa), line => Line};
+enrich(Report, _) -> Report.
 
-filter(Meta) ->
-    maps:without(?WITHOUT_MERGE, Meta).
+enrich_fmt(Fmt, Args, #{mfa := Mfa, line := Line}) ->
+    {Fmt ++ " mfa: ~s line: ~w", Args ++ [mfa(Mfa), Line]};
+enrich_fmt(Fmt, Args, _) ->
+    {Fmt, Args}.
+
+mfa({M, F, A}) ->
+    <<(atom_to_binary(M, utf8))/binary, $:,
+      (atom_to_binary(F, utf8))/binary, $/,
+      (integer_to_binary(A))/binary>>.

From 5f51f68cc9835020ce3aa1f89100f6cf8d44cfe7 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 21:49:16 +0200
Subject: [PATCH 08/27] fix(mqttsn): do not call emqx_logger:Level directly

---
 apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl
index f890cc774..f1fb4eeb1 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl
@@ -19,6 +19,7 @@
 -behaviour(gen_server).
 
 -include("src/mqttsn/include/emqx_sn.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -export([ start_link/2
         , stop/0
@@ -31,8 +32,6 @@
 -record(state, {gwid, sock, port, addrs, duration, tref}).
 
 -define(DEFAULT_DURATION, 15*60*1000).
--define(LOG(Level, Format, Args),
-        emqx_logger:Level("MQTT-SN(broadcast): " ++ Format, Args)).
 
 %%--------------------------------------------------------------------
 %% API

From b10fb4e95dc405eaad550571ed32dd1a079a422a Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Sat, 31 Jul 2021 22:40:39 +0200
Subject: [PATCH 09/27] refactor: unify all io:format calls to use the same
 macro

---
 apps/emqx/include/logger.hrl                  |  4 ++++
 apps/emqx/src/emqx.erl                        |  4 ++--
 apps/emqx/src/emqx_app.erl                    | 11 +++++----
 apps/emqx/src/emqx_listeners.erl              |  8 +++----
 apps/emqx/src/emqx_rule_actions_trans.erl     |  4 ----
 apps/emqx_dashboard/src/emqx_dashboard.erl    |  5 ++--
 apps/emqx_gateway/src/coap/emqx_coap_impl.erl | 19 +++++++--------
 .../src/exproto/emqx_exproto_impl.erl         | 22 ++++++++---------
 .../src/lwm2m/emqx_lwm2m_coap_server.erl      | 19 +++++++--------
 apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl | 21 ++++++++--------
 .../src/stomp/emqx_stomp_impl.erl             | 24 +++++++++----------
 apps/emqx_management/src/emqx_mgmt_http.erl   |  5 ++--
 .../src/emqx_resource_transform.erl           |  1 -
 .../src/emqx_rule_actions.erl                 |  8 +++----
 .../src/emqx_rule_registry.erl                | 16 ++++++-------
 15 files changed, 83 insertions(+), 88 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index 2c5bea89f..fac01696d 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -61,4 +61,8 @@
         logger:log(Level, Data, #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
                                  , line => ?LINE})).
 
+%% print to 'user' group leader
+-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
+-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).
+
 -endif.
diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl
index f22b33acc..b1c976cc1 100644
--- a/apps/emqx/src/emqx.erl
+++ b/apps/emqx/src/emqx.erl
@@ -76,8 +76,8 @@ set_debug_secret(PathToSecretFile) ->
                 catch _ : _ -> error({badfile, PathToSecretFile})
                 end;
             {error, Reason} ->
-                io:format("Failed to read debug_info encryption key file ~s: ~p~n",
-                          [PathToSecretFile, Reason]),
+                ?ULOG("Failed to read debug_info encryption key file ~s: ~p~n",
+                      [PathToSecretFile, Reason]),
                 error(Reason)
         end,
     F = fun(init) -> ok;
diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl
index 671caa822..a35c90747 100644
--- a/apps/emqx/src/emqx_app.erl
+++ b/apps/emqx/src/emqx_app.erl
@@ -27,6 +27,8 @@
         ]).
 
 -include("emqx.hrl").
+-include("emqx_release.hrl").
+-include("logger.hrl").
 
 -define(APP, emqx).
 
@@ -37,7 +39,6 @@
                      , ?MOD_DELAYED_SHARD
                      ]).
 
--include("emqx_release.hrl").
 
 %%--------------------------------------------------------------------
 %% Application callbacks
@@ -113,17 +114,17 @@ start_ce_modules() ->
 print_otp_version_warning() -> ok.
 -else.
 print_otp_version_warning() ->
-    io:format("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n",
-              [?OTP_RELEASE]).
+    ?ULOG("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n",
+          [?OTP_RELEASE]).
 -endif. % OTP_RELEASE
 
 -ifndef(TEST).
 
 print_banner() ->
-    io:format("Starting ~s on node ~s~n", [?APP, node()]).
+    ?ULOG("Starting ~s on node ~s~n", [?APP, node()]).
 
 print_vsn() ->
-    io:format("~s ~s is running now!~n", [get_description(), get_release()]).
+    ?ULOG("~s ~s is running now!~n", [get_description(), get_release()]).
 
 -else. % TEST
 
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 8c7334cbe..845caa988 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -18,6 +18,7 @@
 -module(emqx_listeners).
 
 -include("emqx_mqtt.hrl").
+-include("logger.hrl").
 
 %% APIs
 -export([ list/0
@@ -108,14 +109,13 @@ start_listener(ZoneName, ListenerName, #{type := Type, bind := Bind} = Conf) ->
         {error, {already_started, Pid}} ->
             {error, {already_started, Pid}};
         {error, Reason} ->
-            io:format(standard_error, "Failed to start ~s listener ~s on ~s: ~0p~n",
-                      [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]),
+            ?ELOG("Failed to start ~s listener ~s on ~s: ~0p~n",
+                  [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]),
             error(Reason)
     end.
 
 -ifndef(TEST).
-console_print(Fmt, Args) ->
-    io:format(Fmt, Args).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
 -else.
 console_print(_Fmt, _Args) -> ok.
 -endif.
diff --git a/apps/emqx/src/emqx_rule_actions_trans.erl b/apps/emqx/src/emqx_rule_actions_trans.erl
index 9ee21385b..df1e58797 100644
--- a/apps/emqx/src/emqx_rule_actions_trans.erl
+++ b/apps/emqx/src/emqx_rule_actions_trans.erl
@@ -18,18 +18,14 @@ trans([Form | AST], ResAST) ->
   trans(AST, [Form | ResAST]).
 
 trans_func_clauses("on_action_create_" ++ _ = _FuncName , Clauses) ->
-  %io:format("~n[[transing function: ~p]]~n", [_FuncName]),
-  %io:format("~n-----old clauses:~n", []), merl:print(Clauses),
   NewClauses = [
     begin
       Bindings = lists:flatten(get_vars(Args) ++ get_vars(Body, lefth)),
       Body2 = append_to_result(Bindings, Body),
       {clause, LineNo, Args, Guards, Body2}
     end || {clause, LineNo, Args, Guards, Body} <- Clauses],
-  %io:format("~n-----new clauses: ~n"), merl:print(NewClauses),
   NewClauses;
 trans_func_clauses(_FuncName, Clauses) ->
-  %io:format("~n[[discarding function: ~p]]~n", [_FuncName]),
   Clauses.
 
 get_vars(Exprs) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl
index 9eb1b06ac..656283bf6 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard.erl
@@ -27,6 +27,7 @@
 %% Authorization
 -export([authorize_appid/1]).
 
+-include_lib("emqx/include/logger.hrl").
 
 -define(BASE_PATH, "/api/v5").
 
@@ -67,7 +68,7 @@ start_listener({Proto, Port, Options}) ->
         dispatch => Dispatch},
     MinirestOptions = maps:merge(Minirest, RanchOptions),
     {ok, _} = minirest:start(listener_name(Proto), MinirestOptions),
-    io:format("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
+    ?ULOG("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
 
 apps() ->
     [App || {App, _, _} <- application:loaded_applications(),
@@ -90,7 +91,7 @@ ranch_opts(Port, Options0) ->
     maps:from_list([{port, Port} | Options]).
 
 stop_listener({Proto, Port, _}) ->
-    io:format("Stop dashboard listener on ~s successfully.~n",[format(Port)]),
+    ?ULOG("Stop dashboard listener on ~s successfully.~n", [format(Port)]),
     minirest:stop(listener_name(Proto)).
 
 listeners() ->
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
index 62cdd3bf2..6d27cd85a 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
@@ -31,6 +31,8 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+
 -dialyzer({nowarn_function, [load/0]}).
 
 %%--------------------------------------------------------------------
@@ -103,13 +105,12 @@ start_listener(InstaId, Ctx, ResourceMod, {Type, ListenOn, SocketOpts, Cfg}) ->
     Cfg2 = Cfg#{resource => ResourceMod},
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg2) of
         {ok, Pid} ->
-            io:format("Start coap ~s:~s listener on ~s successfully.~n",
-                      [InstaId, Type, ListenOnStr]),
+            ?ULOG("Start coap ~s:~s listener on ~s successfully.~n",
+                  [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to start coap ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]),
+            ?ELOG("Failed to start coap ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -136,13 +137,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop coap ~s:~s listener on ~s successfully.~n",
+        ok -> ?ULOG("Stop coap ~s:~s listener on ~s successfully.~n",
                         [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to stop coap ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
+            ?ELOG("Failed to stop coap ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
     end,
     StopRet.
 
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
index b55d30729..6f47e25ff 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
@@ -32,6 +32,8 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -64,8 +66,7 @@ start_grpc_server(InstaId, Options = #{bind := ListenOn}) ->
                       [{ssl_options, SslOpts}]
               end,
     _ = grpc:start_server(InstaId, ListenOn, Services, SvrOptions),
-    io:format("Start ~s gRPC server on ~p successfully.~n",
-               [InstaId, ListenOn]).
+    ?ULOG("Start ~s gRPC server on ~p successfully.~n", [InstaId, ListenOn]).
 
 start_grpc_client_channel(InstaId, Options = #{address := UriStr}) ->
     UriMap = uri_string:parse(UriStr),
@@ -146,13 +147,12 @@ start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            io:format("Start exproto ~s:~s listener on ~s successfully.~n",
+            ?ULOG("Start exproto ~s:~s listener on ~s successfully.~n",
                       [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to start exproto ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]),
+            ?ELOG("Failed to start exproto ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -204,13 +204,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop exproto ~s:~s listener on ~s successfully.~n",
-                        [InstaId, Type, ListenOnStr]);
+        ok -> ?ULOG("Stop exproto ~s:~s listener on ~s successfully.~n",
+                    [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to stop exproto ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
+            ?ELOG("Failed to stop exproto ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
     end,
     StopRet.
 
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
index 7b6aa86af..b2f793364 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
@@ -28,8 +28,7 @@
         , stop_listener/2
         ]).
 
--define(LOG(Level, Format, Args),
-    logger:Level("LwM2M: " ++ Format, Args)).
+-include_lib("emqx/include/logger.hrl").
 
 start(Envs) ->
     start_listeners(Envs).
@@ -50,11 +49,11 @@ stop_listeners(Envs) ->
 start_listener({Proto, ListenOn, Opts}) ->
     case start_listener(Proto, ListenOn, Opts) of
         {ok, _Pid} ->
-            io:format("Start lwm2m:~s listener on ~s successfully.~n",
-                      [Proto, format(ListenOn)]);
+            ?ULOG("Start lwm2m:~s listener on ~s successfully.~n",
+                  [Proto, format(ListenOn)]);
         {error, Reason} ->
-            io:format(standard_error, "Failed to start lwm2m:~s listener on ~s: ~0p~n",
-                      [Proto, format(ListenOn), Reason]),
+            ?ELOG("Failed to start lwm2m:~s listener on ~s: ~0p~n",
+                  [Proto, format(ListenOn), Reason]),
             error(Reason)
     end.
 
@@ -66,11 +65,11 @@ start_listener(dtls, ListenOn, Opts) ->
 stop_listener({Proto, ListenOn, _Opts}) ->
     Ret = stop_listener(Proto, ListenOn),
     case Ret of
-        ok -> io:format("Stop lwm2m:~s listener on ~s successfully.~n",
-                        [Proto, format(ListenOn)]);
+        ok -> ?ULOG("Stop lwm2m:~s listener on ~s successfully.~n",
+                    [Proto, format(ListenOn)]);
         {error, Reason} ->
-            io:format(standard_error, "Failed to stop lwm2m:~s listener on ~s: ~0p~n",
-                      [Proto, format(ListenOn), Reason])
+            ?ELOG("Failed to stop lwm2m:~s listener on ~s: ~0p~n",
+                  [Proto, format(ListenOn), Reason])
     end,
     Ret.
 
diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
index bd27c9fb5..57070206c 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
@@ -32,6 +32,8 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -113,13 +115,12 @@ start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            io:format("Start mqttsn ~s:~s listener on ~s successfully.~n",
-                      [InstaId, Type, ListenOnStr]),
+            ?ULOG("Start mqttsn ~s:~s listener on ~s successfully.~n",
+                  [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to start mqttsn ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]),
+            ?ELOG("Failed to start mqttsn ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -150,13 +151,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop mqttsn ~s:~s listener on ~s successfully.~n",
-                        [InstaId, Type, ListenOnStr]);
+        ok -> ?ULOG("Stop mqttsn ~s:~s listener on ~s successfully.~n",
+                    [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to stop mqttsn ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
+            ?ELOG("Failed to stop mqttsn ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
     end,
     StopRet.
 
diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
index 64d0b32e5..95183ad5e 100644
--- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
+++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
@@ -16,8 +16,6 @@
 
 -module(emqx_stomp_impl).
 
--include_lib("emqx_gateway/include/emqx_gateway.hrl").
-
 -behavior(emqx_gateway_impl).
 
 %% APIs
@@ -31,6 +29,9 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx_gateway/include/emqx_gateway.hrl").
+-include_lib("emqx/include/logger.hrl").
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -97,13 +98,12 @@ start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            io:format("Start stomp ~s:~s listener on ~s successfully.~n",
-                      [InstaId, Type, ListenOnStr]),
+            ?ULOG("Start stomp ~s:~s listener on ~s successfully.~n",
+                  [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to start stomp ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]),
+            ?ELOG("Failed to start stomp ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -134,13 +134,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop stomp ~s:~s listener on ~s successfully.~n",
-                        [InstaId, Type, ListenOnStr]);
+        ok -> ?ULOG("Stop stomp ~s:~s listener on ~s successfully.~n",
+                    [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
-                      "Failed to stop stomp ~s:~s listener on ~s: ~0p~n",
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
+            ?ELOG("Failed to stop stomp ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
     end,
     StopRet.
 
diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl
index 85165412c..8c7b0d7c5 100644
--- a/apps/emqx_management/src/emqx_mgmt_http.erl
+++ b/apps/emqx_management/src/emqx_mgmt_http.erl
@@ -24,6 +24,7 @@
 -export([authorize_appid/1]).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -define(APP, emqx_management).
 
@@ -63,7 +64,7 @@ start_listener({Proto, Port, Options}) ->
         swagger_global_spec => GlobalSpec},
     MinirestOptions = maps:merge(Minirest, RanchOptions),
     {ok, _} = minirest:start(listener_name(Proto), MinirestOptions),
-    io:format("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
+    ?ULOG("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
 
 ranch_opts(Port, Options0) ->
     Options = lists:foldl(
@@ -79,7 +80,7 @@ ranch_opts(Port, Options0) ->
     maps:from_list([{port, Port} | Options]).
 
 stop_listener({Proto, Port, _}) ->
-    io:format("Stop http:management listener on ~s successfully.~n",[format(Port)]),
+    ?ULOG("Stop http:management listener on ~s successfully.~n",[format(Port)]),
     minirest:stop(listener_name(Proto)).
 
 listeners() ->
diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl
index a738a4ca6..ac632cd45 100644
--- a/apps/emqx_resource/src/emqx_resource_transform.erl
+++ b/apps/emqx_resource/src/emqx_resource_transform.erl
@@ -59,7 +59,6 @@ form(Mod, Form) ->
         ?Q("-module('@_').") ->
             {[Form], fix_spec_attrs(), fix_spec_funcs(Mod)};
         _ ->
-            %io:format("---other form: ~p~n", [Form]),
             {[Form], [], []}
     end.
 
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index 61dc36b35..ad035356f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -119,10 +119,10 @@ on_action_create_inspect(Id, Params) ->
 
 -spec on_action_inspect(selected_data(), env_vars()) -> any().
 on_action_inspect(Selected, Envs) ->
-    io:format("[inspect]~n"
-              "\tSelected Data: ~p~n"
-              "\tEnvs: ~p~n"
-              "\tAction Init Params: ~p~n", [Selected, Envs, ?bound_v('Params', Envs)]),
+    ?ULOG("[inspect]~n"
+          "\tSelected Data: ~p~n"
+          "\tEnvs: ~p~n"
+          "\tAction Init Params: ~p~n", [Selected, Envs, ?bound_v('Params', Envs)]),
     emqx_rule_metrics:inc_actions_success(?bound_v('Id', Envs)).
 
 
diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl
index f2d717dba..d335a601b 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl
@@ -150,14 +150,14 @@ mnesia(copy) ->
     ok = ekka_mnesia:copy_table(?RES_TYPE_TAB, ram_copies).
 
 dump() ->
-    io:format("Rules: ~p~n"
-              "ActionInstParams: ~p~n"
-              "Resources: ~p~n"
-              "ResourceParams: ~p~n",
-            [ets:tab2list(?RULE_TAB),
-             ets:tab2list(?ACTION_INST_PARAMS_TAB),
-             ets:tab2list(?RES_TAB),
-             ets:tab2list(?RES_PARAMS_TAB)]).
+    ?ULOG("Rules: ~p~n"
+          "ActionInstParams: ~p~n"
+          "Resources: ~p~n"
+          "ResourceParams: ~p~n",
+          [ets:tab2list(?RULE_TAB),
+           ets:tab2list(?ACTION_INST_PARAMS_TAB),
+           ets:tab2list(?RES_TAB),
+           ets:tab2list(?RES_PARAMS_TAB)]).
 
 %%------------------------------------------------------------------------------
 %% Start the registry

From 2fd0742d2ba5b03abafacdb486c3d20901b268da Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 2 Aug 2021 20:17:53 +0200
Subject: [PATCH 10/27] refactor(emqx_ctl): simplify emqx_ctl

---
 bin/emqx     | 13 +++++++-
 bin/emqx_ctl | 83 ++--------------------------------------------------
 2 files changed, 14 insertions(+), 82 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index bc1d00e35..00c50362e 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -98,7 +98,7 @@ relx_usage() {
             echo "                      don't make it permanent"
             ;;
         *)
-            echo "Usage: $REL_NAME {start|start_boot <file>|ertspath|foreground|stop|restart|reboot|pid|ping|console|console_clean|console_boot <file>|attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|rpc|rpcterms|eval|root_dir}"
+            echo "Usage: $REL_NAME {start|start_boot <file>|ertspath|foreground|stop|restart|reboot|pid|ping|console|console_clean|console_boot <file>|attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|ctl|rpc|rpcterms|eval|root_dir}"
             ;;
     esac
 }
@@ -652,6 +652,17 @@ case "$1" in
     ertspath)
         echo "$ERTS_PATH"
         ;;
+    ctl)
+        # Make sure a node IS running
+        if ! relx_nodetool "ping" > /dev/null; then
+            echo "Node is not running!"
+            exit 1
+        fi
+
+        shift
+
+        relx_nodetool rpc emqx_ctl run_command "$@"
+        ;;
     rpc)
         # Make sure a node IS running
         if ! relx_nodetool "ping" > /dev/null; then
diff --git a/bin/emqx_ctl b/bin/emqx_ctl
index d4cdef389..f94946840 100755
--- a/bin/emqx_ctl
+++ b/bin/emqx_ctl
@@ -2,84 +2,5 @@
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
-set -e
-
-ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
-# shellcheck disable=SC1090
-. "$ROOT_DIR"/releases/emqx_vars
-
-export ROOTDIR="$RUNNER_ROOT_DIR"
-export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN"
-export BINDIR="$ERTS_DIR/bin"
-
-export RUNNER_ROOT_DIR
-export REL_VSN
-
-# shellcheck disable=SC2012,SC2086
-LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
-if [ -z "$LATEST_VM_ARGS" ]; then
-    echo "No vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
-    exit 1
-fi
-
-# Echo to stderr on errors
-echoerr() { echo "$@" 1>&2; }
-
-if [ -z "$WITH_EPMD" ]; then
-    EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
-else
-    EPMD_ARG="-start_epmd true"
-fi
-
-call_hocon() {
-    export RUNNER_ROOT_DIR
-    export RUNNER_ETC_DIR
-    "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \
-        || die "ERROR: call_hocon failed: $*" $?
-}
-
-# Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460
-PROTO_DIST="$(call_hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get cluster.proto_dist | tr -d \")"
-if [ -z "$PROTO_DIST" ]; then
-    PROTO_DIST_ARG=""
-else
-    PROTO_DIST_ARG="-proto_dist $PROTO_DIST"
-fi
-
-
-# Control a node
-relx_nodetool() {
-    command="$1"; shift
-
-    ERL_FLAGS="$ERL_FLAGS $EPMD_ARG $PROTO_DIST_ARG" \
-    "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
-                                -setcookie "$COOKIE" "$command" "$@"
-}
-
-NAME="${EMQX_NODE_NAME:-}"
-[ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NAME="${EMQX_NAME}@${EMQX_HOST}"
-[ -z "$NAME" ] && NAME="$(grep -E '^-s?name' "$LATEST_VM_ARGS" | awk '{print $2}')"
-if [ -z "$NAME" ]; then
-    echoerr "Failed to read node name from $LATEST_VM_ARGS"
-    echoerr "Ensure the node is running and you have permission to read $LATEST_VM_ARGS"
-    exit 1
-fi
-
-case "$NAME" in
-    *@*)
-        NAME_TYPE='-name'
-       ;;
-    *)
-        NAME_TYPE='-sname'
-esac
-
-COOKIE="${EMQX_NODE_COOKIE:-}"
-[ -z "$COOKIE" ] && COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
-if [ -z "$COOKIE" ]; then
-    echoerr "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE_COOKIE"
-    exit 1
-fi
-
-cd "$ROOTDIR"
-
-relx_nodetool rpc emqx_ctl run_command "$@"
+THIS_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")" || true; pwd -P)"
+exec "$THIS_DIR/emqx" ctl "$@"

From fb716ae8b2e8f02ffbbb32f6e7b4d623beb28b43 Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Tue, 3 Aug 2021 09:21:07 +0800
Subject: [PATCH 11/27] chore(authn rename): rename emqx_authn to
 authentication

---
 apps/emqx_authn/etc/emqx_authn.conf       | 2 +-
 apps/emqx_authn/src/emqx_authn_api.erl    | 4 ++--
 apps/emqx_authn/src/emqx_authn_app.erl    | 4 ++--
 apps/emqx_authn/src/emqx_authn_schema.erl | 4 ++--
 extension_schemas.config                  | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf
index 8af40fe8f..5ddeb7a32 100644
--- a/apps/emqx_authn/etc/emqx_authn.conf
+++ b/apps/emqx_authn/etc/emqx_authn.conf
@@ -1,4 +1,4 @@
-emqx_authn: {
+authentication: {
     enable: false
     authenticators: [
         # {
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index c0071114a..9477c3e3c 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -1158,12 +1158,12 @@ authentication(post, Request) ->
 authenticators(post, Request) ->
     {ok, Body, _} = cowboy_req:read_body(Request),
     AuthenticatorConfig = emqx_json:decode(Body, [return_maps]),
-    Config = #{<<"emqx_authn">> => #{
+    Config = #{<<"authentication">> => #{
                    <<"authenticators">> => [AuthenticatorConfig]
                }},
     NConfig = hocon_schema:check_plain(emqx_authn_schema, Config,
                                        #{nullable => true}),
-    #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
+    #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
     case emqx_authn:create_authenticator(?CHAIN, NAuthenticatorConfig) of
         {ok, Authenticator2} ->
             {201, Authenticator2};
diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl
index b55727a00..52e59b2a6 100644
--- a/apps/emqx_authn/src/emqx_authn_app.erl
+++ b/apps/emqx_authn/src/emqx_authn_app.erl
@@ -36,8 +36,8 @@ stop(_State) ->
     ok.
 
 initialize() ->
-    AuthNConfig = emqx_config:get([emqx_authn], #{enable => false,
-                                                  authenticators => []}),
+    AuthNConfig = emqx_config:get([authentication], #{enable => false,
+                                                      authenticators => []}),
     initialize(AuthNConfig).
 
 initialize(#{enable := Enable, authenticators := AuthenticatorsConfig}) ->
diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl
index 694d5c7ca..6ef742e65 100644
--- a/apps/emqx_authn/src/emqx_authn_schema.erl
+++ b/apps/emqx_authn/src/emqx_authn_schema.erl
@@ -28,9 +28,9 @@
 -export([ authenticator_name/1
         ]).
 
-structs() -> [ "emqx_authn" ].
+structs() -> [ "authentication" ].
 
-fields("emqx_authn") ->
+fields("authentication") ->
     [ {enable, fun enable/1}
     , {authenticators, fun authenticators/1}
     ].
diff --git a/extension_schemas.config b/extension_schemas.config
index a0a298c1b..1f3b140d8 100644
--- a/extension_schemas.config
+++ b/extension_schemas.config
@@ -5,7 +5,7 @@
 %%
 [ {"emqx_data_bridge", emqx_data_bridge_schema}
 , {"emqx_retainer", emqx_retainer_schema}
-, {"emqx_authn", emqx_authn_schema}
+, {"authentication", emqx_authn_schema}
 , {"authorization", emqx_authz_schema}
 , {"emqx_bridge_mqtt", emqx_bridge_mqtt_schema}
 , {"emqx_management", emqx_management_schema}

From 5e9ed72975334b4ebd18777165a14fbd5ccb1c91 Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Tue, 3 Aug 2021 09:24:49 +0800
Subject: [PATCH 12/27] chore(authn rename): missing name change

---
 apps/emqx_authn/src/emqx_authn_api.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index 9477c3e3c..fe53629de 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -1186,12 +1186,12 @@ authenticators2(put, Request) ->
     AuthenticatorID = cowboy_req:binding(id, Request),
     {ok, Body, _} = cowboy_req:read_body(Request),
     AuthenticatorConfig = emqx_json:decode(Body, [return_maps]),
-    Config = #{<<"emqx_authn">> => #{
+    Config = #{<<"authentication">> => #{
                    <<"authenticators">> => [AuthenticatorConfig]
                }},
     NConfig = hocon_schema:check_plain(emqx_authn_schema, Config,
                                        #{nullable => true}),
-    #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
+    #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
     case emqx_authn:update_or_create_authenticator(?CHAIN, AuthenticatorID, NAuthenticatorConfig) of
         {ok, Authenticator} ->
             {200, Authenticator};

From 149ef6d7cc64a898b739003731c87e9d6c5e5a4a Mon Sep 17 00:00:00 2001
From: lafirest <blankalupo@163.com>
Date: Mon, 2 Aug 2021 16:55:57 +0800
Subject: [PATCH 13/27] style(emqx_gateway): improve some emqx_coap code

---
 apps/emqx_gateway/etc/emqx_gateway.conf       |   4 +-
 .../src/coap/emqx_coap_channel.erl            |  53 +++----
 .../emqx_gateway/src/coap/emqx_coap_frame.erl |  24 +--
 .../src/coap/emqx_coap_observe_res.erl        |  19 +--
 .../src/coap/emqx_coap_session.erl            |  74 ++++-----
 apps/emqx_gateway/src/coap/emqx_coap_tm.erl   | 143 +++++++++---------
 .../src/coap/emqx_coap_transport.erl          |  42 ++---
 .../src/coap/include/emqx_coap.hrl            |   2 +-
 8 files changed, 179 insertions(+), 182 deletions(-)

diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf
index b6dbff834..cdb776af1 100644
--- a/apps/emqx_gateway/etc/emqx_gateway.conf
+++ b/apps/emqx_gateway/etc/emqx_gateway.conf
@@ -36,7 +36,7 @@ gateway: {
       subscribe_qos: qos0
       publish_qos: qos1
       listener.udp.1: {
-        bind: 5687
+        bind: 5683
         }
       }
 
@@ -49,7 +49,7 @@ gateway: {
       subscribe_qos: qos2
       publish_qos: coap
       listener.udp.1: {
-        bind: 5683
+        bind: 5687
         }
       }
 
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
index d5b9b7293..c208d00f8 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
@@ -67,9 +67,9 @@
 -define(DISCONNECT_WAIT_TIME, timer:seconds(10)).
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]).
 
-%%%===================================================================
-%%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
 
 info(Channel) ->
     maps:from_list(info(?INFO_KEYS, Channel)).
@@ -101,7 +101,7 @@ init(ConnInfo = #{peername := {PeerHost, _},
     ClientInfo = set_peercert_infos(
                    Peercert,
                    #{ zone => default
-                    , protocol => 'mqtt-coap'
+                    , protocol => 'coap'
                     , peerhost => PeerHost
                     , sockport => SockPort
                     , clientid => emqx_guid:to_base62(emqx_guid:gen())
@@ -132,8 +132,8 @@ auth_subscribe(Topic,
                  clientinfo := ClientInfo}) ->
     emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic).
 
-transfer_result(Result, From, Value) ->
-    ?TRANSFER_RESULT(Result, [out], From, Value).
+transfer_result(From, Value, Result) ->
+    ?TRANSFER_RESULT([out], From, Value, Result).
 
 %%--------------------------------------------------------------------
 %% Handle incoming packet
@@ -147,17 +147,17 @@ handle_in(#coap_message{method = post,
         <<>> ->
             handle_command(Msg, Channel);
         _ ->
-            call_session(Channel, received, [Msg])
+            call_session(received, [Msg], Channel)
     end;
 
 handle_in(Msg, Channel) ->
-    call_session(ensure_keepalive_timer(Channel), received, [Msg]).
+    call_session(received, [Msg], ensure_keepalive_timer(Channel)).
 
 %%--------------------------------------------------------------------
 %% Handle Delivers from broker to client
 %%--------------------------------------------------------------------
 handle_deliver(Delivers, Channel) ->
-    call_session(Channel, deliver, [Delivers]).
+    call_session(deliver, [Delivers], Channel).
 
 %%--------------------------------------------------------------------
 %% Handle timeout
@@ -165,14 +165,14 @@ handle_deliver(Delivers, Channel) ->
 handle_timeout(_, {keepalive, NewVal}, #channel{keepalive = KeepAlive} = Channel) ->
     case emqx_keepalive:check(NewVal, KeepAlive) of
         {ok, NewKeepAlive} ->
-            Channel2 = ensure_keepalive_timer(Channel, fun make_timer/4),
+            Channel2 = ensure_keepalive_timer(fun make_timer/4, Channel),
             {ok, Channel2#channel{keepalive = NewKeepAlive}};
         {error, timeout} ->
             {shutdown, timeout, Channel}
     end;
 
 handle_timeout(_, {transport, Msg}, Channel) ->
-    call_session(Channel, timeout, [Msg]);
+    call_session(timeout, [Msg], Channel);
 
 handle_timeout(_, disconnect, Channel) ->
     {shutdown, normal, Channel};
@@ -207,9 +207,9 @@ handle_info(Info, Channel) ->
 terminate(_Reason, _Channel) ->
     ok.
 
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
 set_peercert_infos(NoSSL, ClientInfo)
   when NoSSL =:= nossl;
        NoSSL =:= undefined ->
@@ -232,9 +232,9 @@ make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = Timers#{Name => TRef}}.
 
 ensure_keepalive_timer(Channel) ->
-    ensure_keepalive_timer(Channel, fun ensure_timer/4).
+    ensure_keepalive_timer(fun ensure_timer/4, Channel).
 
-ensure_keepalive_timer(#channel{config = Cfg} = Channel, Fun) ->
+ensure_keepalive_timer(Fun, #channel{config = Cfg} = Channel) ->
     Interval = maps:get(heartbeat, Cfg),
     Fun(keepalive, Interval, keepalive, Channel).
 
@@ -285,9 +285,9 @@ run_conn_hooks(Input, Channel = #channel{ctx = Ctx,
                                          conninfo = ConnInfo}) ->
     ConnProps = #{},
     case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of
-                   Error = {error, _Reason} -> Error;
-                   _NConnProps ->
-                       {ok, Input, Channel}
+        Error = {error, _Reason} -> Error;
+        _NConnProps ->
+            {ok, Input, Channel}
     end.
 
 enrich_clientinfo({Queries, Msg},
@@ -339,11 +339,10 @@ ensure_connected(Channel = #channel{ctx = Ctx,
     Channel#channel{conninfo = NConnInfo}.
 
 process_connect(Channel = #channel{ctx = Ctx,
-                                   session = Session,
                                    conninfo = ConnInfo,
                                    clientinfo = ClientInfo},
                 Msg) ->
-    SessFun = fun(_,_) -> Session end,
+    SessFun = fun(_,_) -> emqx_coap_session:new() end,
     case emqx_gateway_ctx:open_session(
            Ctx,
            true,
@@ -367,14 +366,16 @@ run_hooks(Ctx, Name, Args, Acc) ->
     emqx_hooks:run_fold(Name, Args, Acc).
 
 reply(Channel, Method, Payload, Req) ->
-    call_session(Channel, reply, [Req, Method, Payload]).
+    call_session(reply, [Req, Method, Payload], Channel).
 
 ack(Channel, Method, Payload, Req) ->
-    call_session(Channel, piggyback, [Req, Method, Payload]).
+    call_session(piggyback, [Req, Method, Payload], Channel).
 
-call_session(#channel{session = Session,
-                      config = Cfg} = Channel, F, A) ->
-    case erlang:apply(emqx_coap_session, F, [Session, Cfg | A]) of
+call_session(F,
+             A,
+             #channel{session = Session,
+                      config = Cfg} = Channel) ->
+    case erlang:apply(emqx_coap_session, F, A ++ [Cfg, Session]) of
         #{out := Out,
           session := Session2} ->
             {ok, {outgoing, Out}, Channel#channel{session = Session2}};
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl
index 039190646..9a53f3e01 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl
@@ -54,9 +54,9 @@
 -define(OPTION_PROXY_SCHEME, 39).
 -define(OPTION_SIZE1, 60).
 
-%%%===================================================================
-%%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
 
 initial_parse_state(_) ->
     #{}.
@@ -64,9 +64,9 @@ initial_parse_state(_) ->
 serialize_opts() ->
     #{}.
 
-%%%===================================================================
-%%% serialize_pkt
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% serialize_pkt
+%%--------------------------------------------------------------------
 %% empty message
 serialize_pkt(#coap_message{type = Type, method = undefined, id = MsgId}, _Opts) ->
     <<?VERSION:2, (encode_type(Type)):2, 0:4, 0:3, 0:5, MsgId:16>>;
@@ -223,9 +223,9 @@ method_to_class_code({error, proxying_not_supported}) -> {5, 05};
 method_to_class_code(Method) ->
     erlang:throw({bad_method, Method}).
 
-%%%===================================================================
-%%% parse
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% parse
+%%--------------------------------------------------------------------
 parse(<<?VERSION:2, Type:2, 0:4, 0:3, 0:5, MsgId:16>>, ParseState) ->
     {ok,
      #coap_message{ type = decode_type(Type)
@@ -410,9 +410,9 @@ is_message(#coap_message{}) ->
 is_message(_) ->
     false.
 
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
 -spec is_repeatable_option(message_option_name()) -> boolean().
 is_repeatable_option(if_match) -> true;
 is_repeatable_option(etag) -> true;
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl
index 199ad0658..3cf925448 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl
@@ -17,7 +17,7 @@
 -module(emqx_coap_observe_res).
 
 %% API
--export([ new/0, insert/3, remove/2
+-export([ new_manager/0, insert/3, remove/2
         , res_changed/2, foreach/2]).
 -export_type([manager/0]).
 
@@ -26,6 +26,7 @@
 -type topic() :: binary().
 -type token() :: binary().
 -type seq_id() :: 0 .. ?MAX_SEQ_ID.
+
 -type res() :: #{ token := token()
                 , seq_id := seq_id()
                 }.
@@ -35,12 +36,12 @@
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
--spec new() -> manager().
-new() ->
+-spec new_manager() -> manager().
+new_manager() ->
     #{}.
 
--spec insert(manager(), topic(), token()) -> manager().
-insert(Manager, Topic, Token) ->
+-spec insert(topic(), token(), manager()) -> manager().
+insert(Topic, Token, Manager) ->
     case maps:get(Topic, Manager, undefined) of
         undefined ->
             Manager#{Topic => new_res(Token)};
@@ -48,12 +49,12 @@ insert(Manager, Topic, Token) ->
             Manager
     end.
 
--spec remove(manager(), topic()) -> manager().
-remove(Manager, Topic) ->
+-spec remove(topic(), manager()) -> manager().
+remove(Topic, Manager) ->
     maps:remove(Topic, Manager).
 
--spec res_changed(manager(), topic()) -> undefined | {token(), seq_id(), manager()}.
-res_changed(Manager, Topic) ->
+-spec res_changed(topic(), manager()) -> undefined | {token(), seq_id(), manager()}.
+res_changed(Topic, Manager) ->
     case maps:get(Topic, Manager, undefined) of
         undefined ->
             undefined;
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl
index dac4ac924..8b9eed14c 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl
@@ -47,52 +47,52 @@
 new() ->
     _ = emqx_misc:rand_seed(),
     #session{ transport_manager = emqx_coap_tm:new()
-            , observe_manager = emqx_coap_observe_res:new()
+            , observe_manager = emqx_coap_observe_res:new_manager()
             , next_msg_id = rand:uniform(?MAX_MESSAGE_ID)}.
 
 %%%-------------------------------------------------------------------
 %%% Process Message
 %%%-------------------------------------------------------------------
-received(Session, Cfg, #coap_message{type = ack} = Msg) ->
-    handle_response(Session, Cfg, Msg);
+received(#coap_message{type = ack} = Msg, Cfg, Session) ->
+    handle_response(Msg, Cfg, Session);
 
-received(Session, Cfg, #coap_message{type = reset} = Msg) ->
-    handle_response(Session, Cfg, Msg);
+received(#coap_message{type = reset} = Msg, Cfg, Session) ->
+    handle_response(Msg, Cfg, Session);
 
-received(Session, Cfg, #coap_message{method = Method} = Msg) when is_atom(Method) ->
-    handle_request(Session, Cfg, Msg);
+received(#coap_message{method = Method} = Msg, Cfg, Session) when is_atom(Method) ->
+    handle_request(Msg, Cfg, Session);
 
-received(Session, Cfg, Msg) ->
-    handle_response(Session, Cfg, Msg).
+received(Msg, Cfg, Session) ->
+    handle_response(Msg, Cfg, Session).
 
-reply(Session, Cfg, Req, Method) ->
-    reply(Session, Cfg, Req, Method, <<>>).
+reply(Req, Method, Cfg, Session) ->
+    reply(Req, Method, <<>>, Cfg, Session).
 
-reply(Session, Cfg, Req, Method, Payload) ->
+reply(Req, Method, Payload, Cfg, Session) ->
     Response = emqx_coap_message:response(Method, Payload, Req),
-    handle_out(Session, Cfg, Response).
+    handle_out(Response, Cfg, Session).
 
-ack(Session, Cfg, Req) ->
-    piggyback(Session, Cfg, Req, <<>>).
+ack(Req, Cfg, Session) ->
+    piggyback(Req, <<>>, Cfg, Session).
 
-piggyback(Session, Cfg, Req, Payload) ->
+piggyback(Req, Payload, Cfg, Session) ->
     Response = emqx_coap_message:ack(Req),
     Response2 = emqx_coap_message:set_payload(Payload, Response),
-    handle_out(Session, Cfg, Response2).
+    handle_out(Response2, Cfg, Session).
 
-deliver(Session, Cfg, Delivers) ->
+deliver(Delivers, Cfg, Session) ->
     Fun = fun({_, Topic, Message},
               #{out := OutAcc,
                 session := #session{observe_manager = OM,
                                     next_msg_id = MsgId} = SAcc} = Acc) ->
-                  case emqx_coap_observe_res:res_changed(OM, Topic) of
+                  case emqx_coap_observe_res:res_changed(Topic, OM) of
                       undefined ->
                           Acc;
                       {Token, SeqId, OM2} ->
                           Msg = mqtt_to_coap(Message, MsgId, Token, SeqId, Cfg),
                           SAcc2 = SAcc#session{next_msg_id = next_msg_id(MsgId),
                                                observe_manager = OM2},
-                          #{out := Out} = Result = call_transport_manager(SAcc2, Cfg, Msg, handle_out),
+                          #{out := Out} = Result = call_transport_manager(handle_out, Msg, Cfg, SAcc2),
                           Result#{out := [Out | OutAcc]}
                   end
           end,
@@ -101,35 +101,35 @@ deliver(Session, Cfg, Delivers) ->
                   session => Session},
                 Delivers).
 
-timeout(Session, Cfg, Timer) ->
-    call_transport_manager(Session, Cfg, Timer, ?FUNCTION_NAME).
+timeout(Timer, Cfg, Session) ->
+    call_transport_manager(?FUNCTION_NAME, Timer, Cfg, Session).
 
-transfer_result(Result, From, Value) ->
-    ?TRANSFER_RESULT(Result, [out, subscribe], From, Value).
+transfer_result(From, Value, Result) ->
+    ?TRANSFER_RESULT([out, subscribe], From, Value, Result).
 
 %%%-------------------------------------------------------------------
 %%% Internal functions
 %%%-------------------------------------------------------------------
-handle_request(Session, Cfg, Msg) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+handle_request(Msg, Cfg, Session) ->
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-handle_response(Session, Cfg, Msg) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+handle_response(Msg, Cfg, Session) ->
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-handle_out(Session, Cfg, Msg) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+handle_out(Msg, Cfg, Session) ->
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-call_transport_manager(#session{transport_manager = TM} = Session,
-                       Cfg,
+call_transport_manager(Fun,
                        Msg,
-                       Fun) ->
+                       Cfg,
+                       #session{transport_manager = TM} = Session) ->
     try
-        Result = emqx_coap_tm:Fun(Msg, TM, Cfg),
+        Result = emqx_coap_tm:Fun(Msg, Cfg, TM),
         {ok, _, Session2} = emqx_misc:pipeline([fun process_tm/2,
                                                 fun process_subscribe/2],
                                                Result,
                                                Session),
-        emqx_coap_channel:transfer_result(Result, session, Session2)
+        emqx_coap_channel:transfer_result(session, Session2, Result)
     catch Type:Reason:Stack ->
             ?ERROR("process transmission with, message:~p failed~n
 Type:~p,Reason:~p~n,StackTrace:~p~n", [Msg, Type, Reason, Stack]),
@@ -146,10 +146,10 @@ process_subscribe(#{subscribe := Sub}, #session{observe_manager = OM} =  Session
         undefined ->
             {ok, Session};
         {Topic, Token} ->
-            OM2 = emqx_coap_observe_res:insert(OM, Topic, Token),
+            OM2 = emqx_coap_observe_res:insert(Topic, Token, OM),
             {ok, Session#session{observe_manager = OM2}};
         Topic ->
-            OM2 = emqx_coap_observe_res:remove(OM, Topic),
+            OM2 = emqx_coap_observe_res:remove(Topic, OM),
             {ok, Session#session{observe_manager = OM2}}
     end;
 process_subscribe(_, Session) ->
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl
index 677292529..8830d7447 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl
@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% transport manager
+%% the transport state machine manager
 -module(emqx_coap_tm).
 
 -export([ new/0
@@ -23,23 +23,23 @@
         , handle_out/3
         , timeout/3]).
 
--export_type([manager/0, event_result/2]).
+-export_type([manager/0, event_result/1]).
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
 -type direction() :: in | out.
--type transport_id() :: {direction(), non_neg_integer()}.
+-type state_machine_id() :: {direction(), non_neg_integer()}.
 
--record(transport, { id :: transport_id()
-                   , state :: atom()
-                   , timers :: maps:map()
-                   , data :: any()}).
--type transport() :: #transport{}.
+-record(state_machine, { id :: state_machine_id()
+                       , state :: atom()
+                       , timers :: maps:map()
+                       , transport :: emqx_coap_transport:transport()}).
+-type state_machine() :: #state_machine{}.
 
 -type message_id() :: 0 .. ?MAX_MESSAGE_ID.
 
--type manager() :: #{message_id() => transport()}.
+-type manager() :: #{message_id() => state_machine()}.
 
 -type ttimeout() :: {state_timeout, pos_integer(), any()}
                   | {stop_timeout, pos_integer()}.
@@ -47,32 +47,31 @@
 -type topic() :: binary().
 -type token() :: binary().
 -type sub_register() :: {topic(), token()} | topic().
-
--type event_result(State, Data) ::
+-type event_result(State) ::
         #{next => State,
           outgoing => emqx_coap_message(),
           timeouts => list(ttimeout()),
           has_sub  => undefined | sub_register(),
-          data => Data}.
+          transport => emqx_coap_transport:transprot()}.
 
-%%%===================================================================
-%%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
 new() ->
     #{}.
 
-handle_request(#coap_message{id = MsgId} = Msg, TM, Cfg) ->
+handle_request(#coap_message{id = MsgId} = Msg, Cfg, TM) ->
     Id = {in, MsgId},
     case maps:get(Id, TM, undefined) of
-                 undefined ->
-            Data = emqx_coap_transport:new(),
-            Transport = new_transport(Id, Data),
-            process_event(in, Msg, TM, Transport, Cfg);
-        TP ->
-            process_event(in, Msg, TM, TP, Cfg)
+        undefined ->
+            Transport = emqx_coap_transport:new(),
+            Machine = new_state_machine(Id, Transport),
+            process_event(in, Msg, TM, Machine, Cfg);
+        Machine ->
+            process_event(in, Msg, TM, Machine, Cfg)
     end.
 
-handle_response(#coap_message{type = Type, id = MsgId} = Msg, TM, Cfg) ->
+handle_response(#coap_message{type = Type, id = MsgId} = Msg, Cfg, TM) ->
     Id = {out, MsgId},
     case maps:get(Id, TM, undefined) of
         undefined ->
@@ -83,56 +82,50 @@ handle_response(#coap_message{type = Type, id = MsgId} = Msg, TM, Cfg) ->
                     #{out => #coap_message{type = reset,
                                            id = MsgId}}
             end;
-        TP ->
-            process_event(in, Msg, TM, TP, Cfg)
+        Machine ->
+            process_event(in, Msg, TM, Machine, Cfg)
     end.
 
-handle_out(#coap_message{id = MsgId} = Msg, TM, Cfg) ->
+handle_out(#coap_message{id = MsgId} = Msg, Cfg, TM) ->
     Id = {out, MsgId},
     case maps:get(Id, TM, undefined) of
         undefined ->
-            Data = emqx_coap_transport:new(),
-            Transport = new_transport(Id, Data),
-            process_event(out, Msg, TM, Transport, Cfg);
+            Transport = emqx_coap_transport:new(),
+            Machine = new_state_machine(Id, Transport),
+            process_event(out, Msg, TM, Machine, Cfg);
         _ ->
             ?WARN("Repeat sending message with id:~p~n", [Id]),
             ?EMPTY_RESULT
     end.
 
-timeout({Id, Type, Msg}, TM, Cfg) ->
+timeout({Id, Type, Msg}, Cfg, TM) ->
     case maps:get(Id, TM, undefined) of
         undefined ->
             ?EMPTY_RESULT;
-        #transport{timers = Timers} = TP ->
+        #state_machine{timers = Timers} = Machine ->
             %% maybe timer has been canceled
             case maps:is_key(Type, Timers) of
                 true ->
-                    process_event(Type, Msg, TM, TP, Cfg);
+                    process_event(Type, Msg, TM, Machine, Cfg);
                 _ ->
                     ?EMPTY_RESULT
             end
     end.
 
 %%--------------------------------------------------------------------
-%% @doc
-%% @spec
-%% @end
+%% Internal functions
 %%--------------------------------------------------------------------
-
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
-new_transport(Id, Data) ->
-    #transport{id = Id,
-               state = idle,
-               timers = #{},
-               data = Data}.
+new_state_machine(Id, Transport) ->
+    #state_machine{id = Id,
+                   state = idle,
+                   timers = #{},
+                   transport = Transport}.
 
 process_event(stop_timeout,
               _,
               TM,
-              #transport{id = Id,
-                         timers = Timers},
+              #state_machine{id = Id,
+                             timers = Timers},
               _) ->
     lists:foreach(fun({_, Ref}) ->
                           emqx_misc:cancel_timer(Ref)
@@ -143,42 +136,42 @@ process_event(stop_timeout,
 process_event(Event,
               Msg,
               TM,
-              #transport{id = Id,
-                         state = State,
-                         data = Data} = TP,
+              #state_machine{id = Id,
+                             state = State,
+                             transport = Transport} = Machine,
               Cfg) ->
-    Result = emqx_coap_transport:State(Event, Msg, Data, Cfg),
-    {ok, _, TP2} = emqx_misc:pipeline([fun process_state_change/2,
-                                       fun process_data_change/2,
-                                       fun process_timeouts/2],
-                                      Result,
-                                      TP),
-    TM2 = TM#{Id => TP2},
-    emqx_coap_session:transfer_result(Result, tm, TM2).
+    Result = emqx_coap_transport:State(Event, Msg, Transport, Cfg),
+    {ok, _, Machine2} = emqx_misc:pipeline([fun process_state_change/2,
+                                            fun process_transport_change/2,
+                                            fun process_timeouts/2],
+                                           Result,
+                                           Machine),
+    TM2 = TM#{Id => Machine2},
+    emqx_coap_session:transfer_result(tm, TM2, Result).
 
-process_state_change(#{next := Next}, TP) ->
-    {ok, cancel_state_timer(TP#transport{state = Next})};
-process_state_change(_, TP) ->
-    {ok, TP}.
+process_state_change(#{next := Next}, Machine) ->
+    {ok, cancel_state_timer(Machine#state_machine{state = Next})};
+process_state_change(_, Machine) ->
+    {ok, Machine}.
 
-cancel_state_timer(#transport{timers = Timers} = TP) ->
+cancel_state_timer(#state_machine{timers = Timers} = Machine) ->
     case maps:get(state_timer, Timers, undefined) of
         undefined ->
-            TP;
+            Machine;
         Ref ->
             _ = emqx_misc:cancel_timer(Ref),
-            TP#transport{timers = maps:remove(state_timer, Timers)}
+            Machine#state_machine{timers = maps:remove(state_timer, Timers)}
     end.
 
-process_data_change(#{data := Data}, TP) ->
-    {ok, TP#transport{data = Data}};
-process_data_change(_, TP) ->
-    {ok, TP}.
+process_transport_change(#{transport := Transport}, Machine) ->
+    {ok, Machine#state_machine{transport = Transport}};
+process_transport_change(_, Machine) ->
+    {ok, Machine}.
 
-process_timeouts(#{timeouts := []}, TP) ->
-    {ok, TP};
+process_timeouts(#{timeouts := []}, Machine) ->
+    {ok, Machine};
 process_timeouts(#{timeouts := Timeouts},
-                 #transport{id = Id, timers = Timers} = TP) ->
+                 #state_machine{id = Id, timers = Timers} = Machine) ->
     NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) ->
                                     process_timer(Id, Timer, Acc);
                                ({stop_timeout, I}, Acc) ->
@@ -186,11 +179,11 @@ process_timeouts(#{timeouts := Timeouts},
                             end,
                             Timers,
                             Timeouts),
-    {ok, TP#transport{timers = NewTimers}};
+    {ok, Machine#state_machine{timers = NewTimers}};
 
-process_timeouts(_, TP) ->
-    {ok, TP}.
+process_timeouts(_, Machine) ->
+    {ok, Machine}.
 
 process_timer(Id, {Type, Interval, Msg}, Timers) ->
-    Ref = emqx_misc:start_timer(Interval, {transport, {Id, Type, Msg}}),
+    Ref = emqx_misc:start_timer(Interval, {state_machine, {Id, Type, Msg}}),
     Timers#{Type => Ref}.
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl
index 7363b6254..b4c8ae333 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl
@@ -9,21 +9,23 @@
 -define(EXCHANGE_LIFETIME, 247000).
 -define(NON_LIFETIME, 145000).
 
--record(data, { cache :: undefined | emqx_coap_message()
-              , retry_interval :: non_neg_integer()
-              , retry_count :: non_neg_integer()
-              }).
+-record(transport, { cache :: undefined | emqx_coap_message()
+                   , retry_interval :: non_neg_integer()
+                   , retry_count :: non_neg_integer()
+                   }).
 
--type data() :: #data{}.
+-type transport() :: #transport{}.
 
 -export([ new/0, idle/4, maybe_reset/4
         , maybe_resend/4, wait_ack/4, until_stop/4]).
 
--spec new() -> data().
+-export_type([transport/0]).
+
+-spec new() -> transport().
 new() ->
-    #data{cache = undefined,
-          retry_interval = 0,
-          retry_count = 0}.
+    #transport{cache = undefined,
+               retry_interval = 0,
+               retry_count = 0}.
 
 idle(in,
      #coap_message{type = non, id = MsgId, method = Method} = Msg,
@@ -50,14 +52,14 @@ idle(in,
      #coap_message{id = MsgId,
                    type = con,
                    method = Method} = Msg,
-     Data,
+     Transport,
      #{resource := Resource} = Cfg) ->
     Ret = #{next => maybe_resend,
             timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]},
     case Method of
         undefined ->
             ResetMsg = #coap_message{type = reset, id = MsgId},
-            Ret#{data => Data#data{cache = ResetMsg},
+            Ret#{transport => Transport#transport{cache = ResetMsg},
                  out  => ResetMsg};
         _ ->
             {RetMsg, SubInfo} =
@@ -72,7 +74,7 @@ idle(in,
                 end,
             RetMsg2 = RetMsg#coap_message{type = ack},
             Ret#{out => RetMsg2,
-                 data => Data#data{cache = RetMsg2},
+                 transport => Transport#transport{cache = RetMsg2},
                  subscribe => SubInfo}
     end;
 
@@ -81,11 +83,11 @@ idle(out, #coap_message{type = non} = Msg, _, _) ->
       out => Msg,
       timeouts => [{stop_timeout, ?NON_LIFETIME}]};
 
-idle(out, Msg, Data, _) ->
+idle(out, Msg, Transport, _) ->
     _ = emqx_misc:rand_seed(),
     Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR),
     #{next => wait_ack,
-      data => Data#data{cache = Msg},
+      transport => Transport#transport{cache = Msg},
       out => Msg,
       timeouts => [ {state_timeout, Timeout, ack_timeout}
                   , {stop_timeout, ?EXCHANGE_LIFETIME}]}.
@@ -99,7 +101,7 @@ maybe_reset(in, Message, _, _) ->
     end,
     ?EMPTY_RESULT.
 
-maybe_resend(in, _, _, #data{cache = Cache}) ->
+maybe_resend(in, _, _, #transport{cache = Cache}) ->
     #{out => Cache}.
 
 wait_ack(in, #coap_message{type = Type}, _, _) ->
@@ -115,14 +117,14 @@ wait_ack(in, #coap_message{type = Type}, _, _) ->
 wait_ack(state_timeout,
          ack_timeout,
          _,
-         #data{cache = Msg,
-               retry_interval = Timeout,
-               retry_count = Count} =Data) ->
+         #transport{cache = Msg,
+                    retry_interval = Timeout,
+                    retry_count = Count} =Transport) ->
     case Count < ?MAX_RETRANSMIT of
         true ->
             Timeout2 = Timeout * 2,
-            #{data => Data#data{retry_interval = Timeout2,
-                                retry_count = Count + 1},
+            #{transport => Transport#transport{retry_interval = Timeout2,
+                                               retry_count = Count + 1},
               out => Msg,
               timeouts => [{state_timeout, Timeout2, ack_timeout}]};
         _ ->
diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl
index 0e0b33365..911d10a22 100644
--- a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl
+++ b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl
@@ -23,7 +23,7 @@
 -define(MAXIMUM_MAX_AGE, 4294967295).
 
 -define(EMPTY_RESULT, #{}).
--define(TRANSFER_RESULT(R1, Keys, From, Value),
+-define(TRANSFER_RESULT(Keys, From, Value, R1),
         begin
             R2 = maps:with(Keys, R1),
             R2#{From => Value}

From 99534e7212cecc2b829631cd2cffa165ee14391e Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Tue, 3 Aug 2021 10:25:04 +0800
Subject: [PATCH 14/27] feat(authn api): support query authentication status

---
 apps/emqx_authn/src/emqx_authn.erl     |  9 +++++++++
 apps/emqx_authn/src/emqx_authn_api.erl | 26 +++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl
index 27f3eab10..0f734fb30 100644
--- a/apps/emqx_authn/src/emqx_authn.erl
+++ b/apps/emqx_authn/src/emqx_authn.erl
@@ -20,6 +20,7 @@
 
 -export([ enable/0
         , disable/0
+        , is_enabled/0
         ]).
 
 -export([authenticate/2]).
@@ -84,6 +85,14 @@ disable() ->
     emqx:unhook('client.authenticate', {?MODULE, authenticate, []}),
     ok.
 
+is_enabled() ->
+    Callbacks = emqx_hooks:lookup('client.authenticate'),
+    lists:any(fun({callback, {?MODULE, authenticate, []}, _, _}) ->
+                  true;
+                 (_) ->
+                  false
+              end, Callbacks).
+
 authenticate(Credential, _AuthResult) ->
     case mnesia:dirty_read(?CHAIN_TAB, ?CHAIN) of
         [#chain{authenticators = Authenticators}] ->
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index fe53629de..63c536d5e 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -131,6 +131,27 @@ authentication_api() ->
                 },
                 <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>)
             }
+        },
+        get => #{
+            description => "Get status of authentication",
+            responses => #{
+                <<"200">> => #{
+                    description => <<"OK">>,
+                    content => #{
+                        'application/json' => #{
+                            schema => #{
+                                type => object,
+                                properties => #{
+                                    enabled => #{
+                                        type => boolean,
+                                        example => true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
         }
     },
     {"/authentication", Metadata, authentication}.
@@ -1153,7 +1174,10 @@ authentication(post, Request) ->
             serialize_error({invalid_parameter, enable});
         _ ->
             serialize_error({missing_parameter, enable})
-    end.
+    end;
+authentication(get, _Request) ->
+    Enabled = emqx_authn:is_enabled(),
+    {200, #{enabled => Enabled}}.
 
 authenticators(post, Request) ->
     {ok, Body, _} = cowboy_req:read_body(Request),

From 05b72157e2c680b5df394463fc0113ec7bf00daf Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Tue, 3 Aug 2021 10:40:08 +0800
Subject: [PATCH 15/27] test(authn): add test cases

---
 apps/emqx_authn/test/emqx_authn_SUITE.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl
index 36b0eabf3..92e506d51 100644
--- a/apps/emqx_authn/test/emqx_authn_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl
@@ -101,5 +101,7 @@ t_authenticate(_) ->
                    username => <<"myuser">>,
 			       password => <<"mypass">>},
     ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)),
+    ?assertEqual(false, emqx_authn:is_enabled()),
     emqx_authn:enable(),
+    ?assertEqual(true, emqx_authn:is_enabled()),
     ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo)).

From 879c191e412f5225d60310c53403b3d39747544a Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Sat, 31 Jul 2021 11:29:25 +0800
Subject: [PATCH 16/27] refactor(exhook): adapt to the hocon schmea

---
 apps/emqx_exhook/.gitignore                   |  29 +
 .../src/exhook => emqx_exhook}/README.md      |   0
 apps/emqx_exhook/docs/design-cn.md            | 116 ++++
 apps/emqx_exhook/etc/emqx_exhook.conf         |  14 +
 .../include/emqx_exhook.hrl                   |   2 +-
 apps/emqx_exhook/priv/emqx_exhook.schema      |  38 ++
 .../priv/protos}/exhook.proto                 |   4 +-
 apps/emqx_exhook/rebar.config                 |  49 ++
 .../src}/emqx_exhook.app.src                  |   2 +-
 apps/emqx_exhook/src/emqx_exhook.appup.src    |  33 ++
 .../src}/emqx_exhook.erl                      |   8 +-
 .../src}/emqx_exhook_app.erl                  |   7 +-
 .../src}/emqx_exhook_cli.erl                  |   2 +-
 .../src}/emqx_exhook_handler.erl              |   3 +-
 apps/emqx_exhook/src/emqx_exhook_schema.erl   |  62 ++
 .../src}/emqx_exhook_server.erl               |  46 +-
 .../src}/emqx_exhook_sup.erl                  |   0
 apps/emqx_exhook/test/emqx_exhook_SUITE.erl   |  96 ++++
 .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 339 +++++++++++
 .../test/props/prop_exhook_hooks.erl          | 531 ++++++++++++++++++
 apps/emqx_gateway/etc/emqx_exhook.conf        |  15 -
 .../src/exhook/prop_exhook_hooks.erl          | 531 ------------------
 .../src/exhook/test/emqx_exhook_SUITE.erl     |  97 ----
 .../src/exhook/test/emqx_exhook_demo_svr.erl  | 339 -----------
 rebar.config.erl                              |   1 +
 25 files changed, 1353 insertions(+), 1011 deletions(-)
 create mode 100644 apps/emqx_exhook/.gitignore
 rename apps/{emqx_gateway/src/exhook => emqx_exhook}/README.md (100%)
 create mode 100644 apps/emqx_exhook/docs/design-cn.md
 create mode 100644 apps/emqx_exhook/etc/emqx_exhook.conf
 rename apps/{emqx_gateway/src/exhook => emqx_exhook}/include/emqx_exhook.hrl (96%)
 create mode 100644 apps/emqx_exhook/priv/emqx_exhook.schema
 rename apps/{emqx_gateway/etc/priv => emqx_exhook/priv/protos}/exhook.proto (99%)
 create mode 100644 apps/emqx_exhook/rebar.config
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook.app.src (94%)
 create mode 100644 apps/emqx_exhook/src/emqx_exhook.appup.src
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook.erl (95%)
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook_app.erl (94%)
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook_cli.erl (98%)
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook_handler.erl (99%)
 create mode 100644 apps/emqx_exhook/src/emqx_exhook_schema.erl
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook_server.erl (90%)
 rename apps/{emqx_gateway/src/exhook => emqx_exhook/src}/emqx_exhook_sup.erl (100%)
 create mode 100644 apps/emqx_exhook/test/emqx_exhook_SUITE.erl
 create mode 100644 apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
 create mode 100644 apps/emqx_exhook/test/props/prop_exhook_hooks.erl
 delete mode 100644 apps/emqx_gateway/etc/emqx_exhook.conf
 delete mode 100644 apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl
 delete mode 100644 apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl
 delete mode 100644 apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl

diff --git a/apps/emqx_exhook/.gitignore b/apps/emqx_exhook/.gitignore
new file mode 100644
index 000000000..da1f0db23
--- /dev/null
+++ b/apps/emqx_exhook/.gitignore
@@ -0,0 +1,29 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
+rebar.lock
+data/
+*.conf.rendered
+*.pyc
+.DS_Store
+*.class
+Mnesia.nonode@nohost/
+src/emqx_exhook_pb.erl
+src/emqx_exhook_v_1_hook_provider_client.erl
+src/emqx_exhook_v_1_hook_provider_bhvr.erl
diff --git a/apps/emqx_gateway/src/exhook/README.md b/apps/emqx_exhook/README.md
similarity index 100%
rename from apps/emqx_gateway/src/exhook/README.md
rename to apps/emqx_exhook/README.md
diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md
new file mode 100644
index 000000000..6686e96e3
--- /dev/null
+++ b/apps/emqx_exhook/docs/design-cn.md
@@ -0,0 +1,116 @@
+# 设计
+
+## 动机
+
+在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力：
+
+1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能
+2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能
+
+但在后续的支持中发现许多难以处理的问题：
+
+1. 有大量的编程语言需要支持，需要编写和维护如 Go, JavaScript, Lua.. 等语言的驱动。
+2. `erlport` 使用的操作系统的管道进行通信，这让用户代码只能部署在和 emqx 同一个操作系统上。部署方式受到了极大的限制。
+3. 用户程序的启动参数直接打包到 Broker 中，导致用户开发无法实时的进行调试，单步跟踪等。
+4. `erlport` 会占用 `stdin` `stdout`。
+
+因此，我们计划重构这部分的实现，其中主要的内容是：
+1. 使用 `gRPC` 替换 `erlport`。
+2. 将 `emqx-extension-hook` 重命名为 `emqx-exhook`
+
+
+旧版本的设计：[emqx-extension-hook design in v4.2.0](https://github.com/emqx/emqx-exhook/blob/v4.2.0/docs/design.md)
+
+## 设计
+
+架构如下：
+
+```
+  EMQ X                                    
++========================+                 +========+==========+
+|    ExHook              |                 |        |          |
+|   +----------------+   |      gRPC       | gRPC   |  User's  |
+|   |   gRPC Client  | ------------------> | Server |  Codes   |
+|   +----------------+   |    (HTTP/2)     |        |          |
+|                        |                 |        |          |
++========================+                 +========+==========+
+```
+
+`emqx-exhook` 通过 gRPC 的方式向用户部署的 gRPC 服务发送钩子的请求，并处理其返回的值。
+
+
+和 emqx 原生的钩子一致，emqx-exhook 也按照链式的方式执行：
+
+<img src="https://docs.emqx.net/broker/latest/cn/advanced/assets/chain_of_responsiblity.png" style="zoom:50%;" />
+
+### gRPC 服务示例
+
+用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中：
+
+```protobuff
+syntax = "proto3";
+
+package emqx.exhook.v1;
+
+service HookProvider {
+
+  rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {};
+
+  rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {};
+
+  rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {};
+
+  rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {};
+
+  rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {};
+
+  rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {};
+
+  rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {};
+
+  rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {};
+
+  rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {};
+}
+```
+
+### 配置文件示例
+
+```
+## 配置 gRPC 服务地址 (HTTP)
+##
+## s1 为服务器的名称
+exhook.server.s1.url = http://127.0.0.1:9001
+
+## 配置 gRPC 服务地址 (HTTPS)
+##
+## s2 为服务器名称
+exhook.server.s2.url = https://127.0.0.1:9002
+exhook.server.s2.cacertfile = ca.pem
+exhook.server.s2.certfile = cert.pem
+exhook.server.s2.keyfile = key.pem
+```
diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf
new file mode 100644
index 000000000..3ed499e28
--- /dev/null
+++ b/apps/emqx_exhook/etc/emqx_exhook.conf
@@ -0,0 +1,14 @@
+##====================================================================
+## EMQ X Hooks
+##====================================================================
+
+exhook: {
+    server.default: {
+        url: "http://127.0.0.1:9000"
+        #ssl: {
+        #    cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem"
+        #    certfile: "{{ platform_etc_dir }}/certs/cert.pem"
+        #    keyfile: "{{ platform_etc_dir }}/certs/key.pem"
+        #}
+    }
+}
diff --git a/apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl b/apps/emqx_exhook/include/emqx_exhook.hrl
similarity index 96%
rename from apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl
rename to apps/emqx_exhook/include/emqx_exhook.hrl
index 64131735e..7301fdcbb 100644
--- a/apps/emqx_gateway/src/exhook/include/emqx_exhook.hrl
+++ b/apps/emqx_exhook/include/emqx_exhook.hrl
@@ -25,7 +25,7 @@
       , {'client.connected',    {emqx_exhook_handler, on_client_connected,     []}}
       , {'client.disconnected', {emqx_exhook_handler, on_client_disconnected,  []}}
       , {'client.authenticate', {emqx_exhook_handler, on_client_authenticate,  []}}
-      , {'client.authorize',    {emqx_exhook_handler, on_client_authorize,     []}}
+      , {'client.check_acl',    {emqx_exhook_handler, on_client_check_acl,     []}}
       , {'client.subscribe',    {emqx_exhook_handler, on_client_subscribe,     []}}
       , {'client.unsubscribe',  {emqx_exhook_handler, on_client_unsubscribe,   []}}
       , {'session.created',     {emqx_exhook_handler, on_session_created,      []}}
diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema
new file mode 100644
index 000000000..e5481a3dd
--- /dev/null
+++ b/apps/emqx_exhook/priv/emqx_exhook.schema
@@ -0,0 +1,38 @@
+%%-*- mode: erlang -*-
+
+{mapping, "exhook.server.$name.url", "emqx_exhook.servers", [
+  {datatype, string}
+]}.
+
+{mapping, "exhook.server.$name.ssl.cacertfile", "emqx_exhook.servers", [
+  {datatype, string}
+]}.
+
+{mapping, "exhook.server.$name.ssl.certfile", "emqx_exhook.servers", [
+  {datatype, string}
+]}.
+
+{mapping, "exhook.server.$name.ssl.keyfile", "emqx_exhook.servers", [
+  {datatype, string}
+]}.
+
+{translation, "emqx_exhook.servers", fun(Conf) ->
+  Filter  = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
+  ServerOptions = fun(Prefix) ->
+                      case http_uri:parse(cuttlefish:conf_get(Prefix ++ ".url", Conf)) of
+                          {ok, {http, _, Host, Port, _, _}} ->
+                              [{scheme, http}, {host, Host}, {port, Port}];
+                          {ok, {https, _, Host, Port, _, _}} ->
+                              [{scheme, https}, {host, Host}, {port, Port},
+                               {ssl_options,
+                                 Filter([{ssl, true},
+                                         {certfile, cuttlefish:conf_get(Prefix ++ ".ssl.certfile", Conf, undefined)},
+                                         {keyfile, cuttlefish:conf_get(Prefix ++ ".ssl.keyfile", Conf, undefined)},
+                                         {cacertfile, cuttlefish:conf_get(Prefix ++ ".ssl.cacertfile", Conf, undefined)}
+                                        ])}];
+                          _ -> error(invalid_server_options)
+                      end
+                  end,
+  [{list_to_atom(Name), ServerOptions("exhook.server." ++ Name)}
+   || {["exhook", "server", Name, "url"], _} <- cuttlefish_variable:filter_by_prefix("exhook.server", Conf)]
+end}.
diff --git a/apps/emqx_gateway/etc/priv/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto
similarity index 99%
rename from apps/emqx_gateway/etc/priv/exhook.proto
rename to apps/emqx_exhook/priv/protos/exhook.proto
index 97a011352..d8a8ef918 100644
--- a/apps/emqx_gateway/etc/priv/exhook.proto
+++ b/apps/emqx_exhook/priv/protos/exhook.proto
@@ -127,14 +127,14 @@ message ClientAuthorizeRequest {
 
   ClientInfo clientinfo = 1;
 
-  enum AuthzReqType {
+  enum AuthorizeReqType {
 
     PUBLISH = 0;
 
     SUBSCRIBE = 1;
   }
 
-  AuthzReqType type = 2;
+  AuthorizeReqType type = 2;
 
   string topic = 3;
 
diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config
new file mode 100644
index 000000000..883aad9bd
--- /dev/null
+++ b/apps/emqx_exhook/rebar.config
@@ -0,0 +1,49 @@
+%%-*- mode: erlang -*-
+{plugins,
+ [rebar3_proper,
+  {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}}
+]}.
+
+{deps,
+ [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}}
+]}.
+
+{grpc,
+ [{protos, ["priv/protos"]},
+  {gpb_opts, [{module_name_prefix, "emqx_"},
+              {module_name_suffix, "_pb"}]}
+]}.
+
+{provider_hooks,
+ [{pre, [{compile, {grpc, gen}},
+         {clean, {grpc, clean}}]}
+]}.
+
+{edoc_opts, [{preprocess, true}]}.
+
+{erl_opts, [warn_unused_vars,
+            warn_shadow_vars,
+            warn_unused_import,
+            warn_obsolete_guard,
+            debug_info,
+            {parse_transform}]}.
+
+{xref_checks, [undefined_function_calls, undefined_functions,
+               locals_not_used, deprecated_function_calls,
+               warnings_as_errors, deprecated_functions]}.
+{xref_ignores, [emqx_exhook_pb]}.
+
+{cover_enabled, true}.
+{cover_opts, [verbose]}.
+{cover_export_enabled, true}.
+{cover_excl_mods, [emqx_exhook_pb,
+                   emqx_exhook_v_1_hook_provider_bhvr,
+                   emqx_exhook_v_1_hook_provider_client]}.
+
+{profiles,
+ [{test,
+   [{deps,
+      [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}}
+      ]}
+    ]}
+]}.
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
similarity index 94%
rename from apps/emqx_gateway/src/exhook/emqx_exhook.app.src
rename to apps/emqx_exhook/src/emqx_exhook.app.src
index e703cfe5e..39c408e20 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,6 +1,6 @@
 {application, emqx_exhook,
  [{description, "EMQ X Extension for Hook"},
-  {vsn, "4.3.2"},
+  {vsn, "4.3.3"},
   {modules, []},
   {registered, []},
   {mod, {emqx_exhook_app, []}},
diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src
new file mode 100644
index 000000000..1da31dbbc
--- /dev/null
+++ b/apps/emqx_exhook/src/emqx_exhook.appup.src
@@ -0,0 +1,33 @@
+%% -*-: erlang -*-
+{VSN,
+ [
+    {"4.3.2", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []}
+    ]},
+    {"4.3.1", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
+    ]},
+    {"4.3.0", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
+    ]},
+    {<<".*">>, []}
+ ],
+ [
+    {"4.3.2", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []}
+    ]},
+    {"4.3.1", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
+    ]},
+    {"4.3.0", [
+      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []},
+      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
+    ]},
+    {<<".*">>, []}
+ ]
+}.
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook.erl b/apps/emqx_exhook/src/emqx_exhook.erl
similarity index 95%
rename from apps/emqx_gateway/src/exhook/emqx_exhook.erl
rename to apps/emqx_exhook/src/emqx_exhook.erl
index 3d977d2f1..e993858cf 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook.erl
+++ b/apps/emqx_exhook/src/emqx_exhook.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_exhook).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 
@@ -40,13 +40,13 @@
 list() ->
     [server(Name) || Name <- running()].
 
--spec enable(atom()|string(), list()) -> ok | {error, term()}.
-enable(Name, Opts) ->
+-spec enable(atom()|string(), map()) -> ok | {error, term()}.
+enable(Name, Options) ->
     case lists:member(Name, running()) of
         true ->
             {error, already_started};
         _ ->
-            case emqx_exhook_server:load(Name, Opts) of
+            case emqx_exhook_server:load(Name, Options) of
                 {ok, ServiceState} ->
                     save(Name, ServiceState);
                 {error, Reason} ->
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl
similarity index 94%
rename from apps/emqx_gateway/src/exhook/emqx_exhook_app.erl
rename to apps/emqx_exhook/src/emqx_exhook_app.erl
index d4621f85e..b057ce463 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_app.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_app.erl
@@ -18,7 +18,7 @@
 
 -behaviour(application).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 
 -emqx_plugin(extension).
 
@@ -67,9 +67,10 @@ stop(_State) ->
 %%--------------------------------------------------------------------
 
 load_all_servers() ->
-    lists:foreach(fun({Name, Options}) ->
+    _ = maps:map(fun(Name, Options) ->
         load_server(Name, Options)
-    end, application:get_env(?APP, servers, [])).
+    end, emqx_config:get([exhook, server])),
+    ok.
 
 unload_all_servers() ->
     emqx_exhook:disable_all().
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl
similarity index 98%
rename from apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl
rename to apps/emqx_exhook/src/emqx_exhook_cli.erl
index efce962d9..a8dc43b16 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_cli.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_cli.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_exhook_cli).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 
 -export([cli/1]).
 
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl
similarity index 99%
rename from apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl
rename to apps/emqx_exhook/src/emqx_exhook_handler.erl
index 2bb2dc7f5..d2fb84bab 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_handler.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_exhook_handler).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -87,6 +87,7 @@ on_client_disconnected(ClientInfo, Reason, _ConnInfo) ->
            },
     cast('client.disconnected', Req).
 
+%% FIXME: `AuthResult`
 on_client_authenticate(ClientInfo, AuthResult) ->
     %% XXX: Bool is missing more information about the atom of the result
     %%      So, the `Req` has missed detailed info too.
diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl
new file mode 100644
index 000000000..867c8e6df
--- /dev/null
+++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl
@@ -0,0 +1,62 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_schema).
+
+-dialyzer(no_return).
+-dialyzer(no_match).
+-dialyzer(no_contracts).
+-dialyzer(no_unused).
+-dialyzer(no_fail_call).
+
+-include_lib("typerefl/include/types.hrl").
+
+-behaviour(hocon_schema).
+
+-export([structs/0, fields/1]).
+-export([t/1, t/3, t/4, ref/1]).
+
+structs() -> [server].
+
+fields(server) ->
+    [{"$name", t(ref(server_structs))}];
+
+fields(server_structs) ->
+    [ {url, t(string(), "emqx_exhook.url", "")}
+    , {ssl, t(ref(ssl_conf_group))}
+    ];
+
+fields(ssl_conf_group) ->
+    [ {cacertfile, string()}
+    , {certfile, string()}
+    , {keyfile, string()}
+    ].
+
+%% types
+
+t(Type) -> #{type => Type}.
+
+t(Type, Mapping, Default) ->
+    hoconsc:t(Type, #{mapping => Mapping, default => Default}).
+
+t(Type, Mapping, Default, OverrideEnv) ->
+    hoconsc:t(Type, #{ mapping => Mapping
+                     , default => Default
+                     , override_env => OverrideEnv
+                     }).
+
+ref(Field) ->
+    hoconsc:ref(?MODULE, Field).
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl
similarity index 90%
rename from apps/emqx_gateway/src/exhook/emqx_exhook_server.erl
rename to apps/emqx_exhook/src/emqx_exhook_server.erl
index 79bb9d32b..5bda76d13 100644
--- a/apps/emqx_gateway/src/exhook/emqx_exhook_server.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_server.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_exhook_server).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 
@@ -74,13 +74,17 @@
 
 -export_type([server/0]).
 
+-type options() :: #{ url := uri_string:uri_string()
+                    , ssl => map()
+                    }.
+
 -dialyzer({nowarn_function, [inc_metrics/2]}).
 
 %%--------------------------------------------------------------------
 %% Load/Unload APIs
 %%--------------------------------------------------------------------
 
--spec load(atom(), list()) -> {ok, server()} | {error, term()} .
+-spec load(atom(), options()) -> {ok, server()} | {error, term()} .
 load(Name0, Opts0) ->
     Name = to_list(Name0),
     {SvrAddr, ClientOpts} = channel_opts(Opts0),
@@ -117,20 +121,27 @@ to_list(Name) when is_list(Name) ->
     Name.
 
 %% @private
-channel_opts(Opts) ->
-    Scheme = proplists:get_value(scheme, Opts),
-    Host = proplists:get_value(host, Opts),
-    Port = proplists:get_value(port, Opts),
-    SvrAddr = format_http_uri(Scheme, Host, Port),
-    ClientOpts = case Scheme of
-                     https ->
-                         SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])),
-                         #{gun_opts =>
-                           #{transport => ssl,
-                             transport_opts => SslOpts}};
-                     _ -> #{}
-                 end,
-    {SvrAddr, ClientOpts}.
+channel_opts(Opts = #{url := URL}) ->
+    io:format("~p~n", [Opts]),
+    case uri_string:parse(URL) of
+        #{scheme := <<"http">>, host := Host, port := Port} ->
+            {format_http_uri("http", Host, Port), #{}};
+        #{scheme := <<"https">>, host := Host, port := Port} ->
+            SslOpts =
+                case maps:get(ssl, Opts, undefined) of
+                    undefined -> [];
+                    MapOpts ->
+                        filter(
+                          [{cacertfile, maps:get(cacertfile, MapOpts, undefined)},
+                           {certfile, maps:get(certfile, MapOpts, undefined)},
+                           {keyfile, maps:get(keyfile, MapOpts, undefined)}
+                          ])
+                end,
+            {format_http_uri("https", Host, Port),
+             #{gun_opts => #{transport => ssl, transport_opts => SslOpts}}};
+        _ ->
+            error(bad_server_url)
+    end.
 
 format_http_uri(Scheme, Host0, Port) ->
     Host = case is_tuple(Host0) of
@@ -139,6 +150,9 @@ format_http_uri(Scheme, Host0, Port) ->
            end,
     lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])).
 
+filter(Ls) ->
+    [ E || E <- Ls, E /= undefined].
+
 -spec unload(server()) -> ok.
 unload(#server{name = Name, hookspec = HookSpecs}) ->
     _ = do_deinit(Name),
diff --git a/apps/emqx_gateway/src/exhook/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl
similarity index 100%
rename from apps/emqx_gateway/src/exhook/emqx_exhook_sup.erl
rename to apps/emqx_exhook/src/emqx_exhook_sup.erl
diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
new file mode 100644
index 000000000..5d5a396a5
--- /dev/null
+++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
@@ -0,0 +1,96 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+
+all() -> emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    _ = emqx_exhook_demo_svr:start(),
+    emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
+    Cfg.
+
+end_per_suite(_Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_exhook]),
+    emqx_exhook_demo_svr:stop().
+
+set_special_cfgs(emqx) ->
+    application:set_env(emqx, allow_anonymous, false),
+    application:set_env(emqx, enable_acl_cache, false),
+    application:set_env(emqx, plugins_loaded_file, undefined),
+    application:set_env(emqx, modules_loaded_file, undefined);
+set_special_cfgs(emqx_exhook) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Test cases
+%%--------------------------------------------------------------------
+
+t_noserver_nohook(_) ->
+    emqx_exhook:disable(default),
+    ?assertEqual([], ets:tab2list(emqx_hooks)),
+
+    Opts = proplists:get_value(
+             default,
+             application:get_env(emqx_exhook, servers, [])
+            ),
+    ok = emqx_exhook:enable(default, Opts),
+    ?assertNotEqual([], ets:tab2list(emqx_hooks)).
+
+t_cli_list(_) ->
+    meck_print(),
+    ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]]
+                , emqx_exhook_cli:cli(["server", "list"])
+                ),
+    unmeck_print().
+
+t_cli_enable_disable(_) ->
+    meck_print(),
+    ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
+    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
+    ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])),
+
+    ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
+    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
+    unmeck_print().
+
+t_cli_stats(_) ->
+    meck_print(),
+    _ = emqx_exhook_cli:cli(["server", "stats"]),
+    _ = emqx_exhook_cli:cli(x),
+    unmeck_print().
+
+%%--------------------------------------------------------------------
+%% Utils
+%%--------------------------------------------------------------------
+
+meck_print() ->
+    meck:new(emqx_ctl, [passthrough, no_history, no_link]),
+    meck:expect(emqx_ctl, print, fun(_) -> ok end),
+    meck:expect(emqx_ctl, print, fun(_, Args) -> Args end).
+
+unmeck_print() ->
+    meck:unload(emqx_ctl).
diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
new file mode 100644
index 000000000..c2db04dd4
--- /dev/null
+++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
@@ -0,0 +1,339 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_demo_svr).
+
+-behavior(emqx_exhook_v_1_hook_provider_bhvr).
+
+%%
+-export([ start/0
+        , stop/0
+        , take/0
+        , in/1
+        ]).
+
+%% gRPC server HookProvider callbacks
+-export([ on_provider_loaded/2
+        , on_provider_unloaded/2
+        , on_client_connect/2
+        , on_client_connack/2
+        , on_client_connected/2
+        , on_client_disconnected/2
+        , on_client_authenticate/2
+        , on_client_check_acl/2
+        , on_client_subscribe/2
+        , on_client_unsubscribe/2
+        , on_session_created/2
+        , on_session_subscribed/2
+        , on_session_unsubscribed/2
+        , on_session_resumed/2
+        , on_session_discarded/2
+        , on_session_takeovered/2
+        , on_session_terminated/2
+        , on_message_publish/2
+        , on_message_delivered/2
+        , on_message_dropped/2
+        , on_message_acked/2
+        ]).
+
+-define(PORT, 9000).
+-define(NAME, ?MODULE).
+
+%%--------------------------------------------------------------------
+%% Server APIs
+%%--------------------------------------------------------------------
+
+start() ->
+    Pid = spawn(fun mngr_main/0),
+    register(?MODULE, Pid),
+    {ok, Pid}.
+
+stop() ->
+    grpc:stop_server(?NAME),
+    ?MODULE ! stop.
+
+take() ->
+    ?MODULE ! {take, self()},
+    receive {value, V} -> V
+    after 5000 -> error(timeout) end.
+
+in({FunName, Req}) ->
+    ?MODULE ! {in, FunName, Req}.
+
+mngr_main() ->
+    application:ensure_all_started(grpc),
+    Services = #{protos => [emqx_exhook_pb],
+                 services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
+                },
+    Options = [],
+    Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
+    mngr_loop([Svr, queue:new(), queue:new()]).
+
+mngr_loop([Svr, Q, Takes]) ->
+    receive
+        {in, FunName, Req} ->
+            {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
+            mngr_loop([Svr, NQ1, NQ2]);
+        {take, From} ->
+            {NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
+            mngr_loop([Svr, NQ1, NQ2]);
+        stop ->
+            exit(normal)
+    end.
+
+reply(Q1, Q2) ->
+    case queue:len(Q1) =:= 0 orelse
+         queue:len(Q2) =:= 0 of
+        true -> {Q1, Q2};
+        _ ->
+            {{value, {Name, V}}, NQ1} = queue:out(Q1),
+            {{value, From}, NQ2} = queue:out(Q2),
+            From ! {value, {Name, V}},
+            {NQ1, NQ2}
+    end.
+
+%%--------------------------------------------------------------------
+%% callbacks
+%%--------------------------------------------------------------------
+
+-spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+
+on_provider_loaded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{hooks => [
+                     #{name => <<"client.connect">>},
+                     #{name => <<"client.connack">>},
+                     #{name => <<"client.connected">>},
+                     #{name => <<"client.disconnected">>},
+                     #{name => <<"client.authenticate">>},
+                     #{name => <<"client.check_acl">>},
+                     #{name => <<"client.subscribe">>},
+                     #{name => <<"client.unsubscribe">>},
+                     #{name => <<"session.created">>},
+                     #{name => <<"session.subscribed">>},
+                     #{name => <<"session.unsubscribed">>},
+                     #{name => <<"session.resumed">>},
+                     #{name => <<"session.discarded">>},
+                     #{name => <<"session.takeovered">>},
+                     #{name => <<"session.terminated">>},
+                     #{name => <<"message.publish">>},
+                     #{name => <<"message.delivered">>},
+                     #{name => <<"message.acked">>},
+                     #{name => <<"message.dropped">>}]}, Md}.
+-spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_provider_unloaded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connect(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connack(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connected(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_disconnected(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case Username of
+        <<"baduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, false}}, Md};
+        <<"gooduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, true}}, Md};
+        <<"normaluser">> ->
+            {ok, #{type => 'CONTINUE',
+                   value => {bool_result, true}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_client_check_acl(emqx_exhook_pb:client_check_acl_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_check_acl(#{clientinfo := #{username := Username}} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case Username of
+        <<"baduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, false}}, Md};
+        <<"gooduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, true}}, Md};
+        <<"normaluser">> ->
+            {ok, #{type => 'CONTINUE',
+                   value => {bool_result, true}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_subscribe(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_unsubscribe(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_created(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_subscribed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_unsubscribed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_resumed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_discarded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_takeovered(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_terminated(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_publish(#{message := #{from := From} = Msg} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case From of
+        <<"baduser">> ->
+            NMsg = Msg#{qos => 0,
+                        topic => <<"">>,
+                        payload => <<"">>
+                       },
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {message, NMsg}}, Md};
+        <<"gooduser">> ->
+            NMsg = Msg#{topic => From,
+                        payload => From},
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {message, NMsg}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_delivered(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_dropped(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_acked(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
new file mode 100644
index 000000000..24f45c8b0
--- /dev/null
+++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
@@ -0,0 +1,531 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(prop_exhook_hooks).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-import(emqx_ct_proper_types,
+        [ conninfo/0
+        , clientinfo/0
+        , sessioninfo/0
+        , message/0
+        , connack_return_code/0
+        , topictab/0
+        , topic/0
+        , subopts/0
+        ]).
+
+-define(ALL(Vars, Types, Exprs),
+        ?SETUP(fun() ->
+            State = do_setup(),
+            fun() -> do_teardown(State) end
+         end, ?FORALL(Vars, Types, Exprs))).
+
+%%--------------------------------------------------------------------
+%% Properties
+%%--------------------------------------------------------------------
+
+prop_client_connect() ->
+    ?ALL({ConnInfo, ConnProps},
+         {conninfo(), conn_properties()},
+       begin
+           ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]),
+           {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(),
+           Expected =
+               #{props => properties(ConnProps),
+                 conninfo => from_conninfo(ConnInfo)
+                },
+           ?assertEqual(Expected, Resp),
+           true
+       end).
+
+prop_client_connack() ->
+    ?ALL({ConnInfo, Rc, AckProps},
+         {conninfo(), connack_return_code(), ack_properties()},
+        begin
+            ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]),
+            {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(AckProps),
+                  result_code => atom_to_binary(Rc, utf8),
+                  conninfo => from_conninfo(ConnInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_authenticate() ->
+    ?ALL({ClientInfo0, AuthResult},
+         {clientinfo(), authresult()},
+        begin
+            ClientInfo = inject_magic_into(username, ClientInfo0),
+            OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult),
+            ExpectedAuthResult = case maps:get(username, ClientInfo) of
+                                     <<"baduser">> ->
+                                         AuthResult#{
+                                           auth_result => not_authorized,
+                                           anonymous => false};
+                                     <<"gooduser">> ->
+                                         AuthResult#{
+                                           auth_result => success,
+                                           anonymous => false};
+                                     <<"normaluser">> ->
+                                         AuthResult#{
+                                           auth_result => success,
+                                           anonymous => false};
+                                     _ ->
+                                         case maps:get(auth_result, AuthResult) of
+                                             success ->
+                                                 #{auth_result => success,
+                                                   anonymous => false};
+                                             _ ->
+                                                 #{auth_result => not_authorized,
+                                                   anonymous => false}
+                                         end
+                                 end,
+            ?assertEqual(ExpectedAuthResult, OutAuthResult),
+
+            {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{result => authresult_to_bool(AuthResult),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_check_acl() ->
+    ?ALL({ClientInfo0, PubSub, Topic, Result},
+         {clientinfo(), oneof([publish, subscribe]),
+          topic(), oneof([allow, deny])},
+        begin
+            ClientInfo = inject_magic_into(username, ClientInfo0),
+            OutResult = emqx_hooks:run_fold(
+                          'client.check_acl',
+                          [ClientInfo, PubSub, Topic],
+                          Result),
+            ExpectedOutResult = case maps:get(username, ClientInfo) of
+                                    <<"baduser">> -> deny;
+                                    <<"gooduser">> -> allow;
+                                    <<"normaluser">> -> allow;
+                                    _ -> Result
+                                 end,
+            ?assertEqual(ExpectedOutResult, OutResult),
+
+            {'on_client_check_acl', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{result => aclresult_to_bool(Result),
+                  type => pubsub_to_enum(PubSub),
+                  topic => Topic,
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_connected() ->
+    ?ALL({ClientInfo, ConnInfo},
+         {clientinfo(), conninfo()},
+        begin
+            ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]),
+            {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_disconnected() ->
+    ?ALL({ClientInfo, Reason, ConnInfo},
+         {clientinfo(), shutdown_reason(), conninfo()},
+        begin
+            ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]),
+            {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{reason => stringfy(Reason),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_subscribe() ->
+    ?ALL({ClientInfo, SubProps, TopicTab},
+         {clientinfo(), sub_properties(), topictab()},
+        begin
+            ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]),
+            {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(SubProps),
+                  topic_filters => topicfilters(TopicTab),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_unsubscribe() ->
+    ?ALL({ClientInfo, UnSubProps, TopicTab},
+         {clientinfo(), unsub_properties(), topictab()},
+        begin
+            ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]),
+            {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(UnSubProps),
+                  topic_filters => topicfilters(TopicTab),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_created() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]),
+            {'on_session_created', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+             ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_subscribed() ->
+    ?ALL({ClientInfo, Topic, SubOpts},
+         {clientinfo(), topic(), subopts()},
+        begin
+            ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]),
+            {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{topic => Topic,
+                  subopts => subopts(SubOpts),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_unsubscribed() ->
+    ?ALL({ClientInfo, Topic, SubOpts},
+         {clientinfo(), topic(), subopts()},
+        begin
+            ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]),
+            {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{topic => Topic,
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_resumed() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]),
+            {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_discared() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]),
+            {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_takeovered() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]),
+            {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_terminated() ->
+    ?ALL({ClientInfo, Reason, SessInfo},
+         {clientinfo(), shutdown_reason(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]),
+            {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{reason => stringfy(Reason),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_message_publish() ->
+    ?ALL(Msg0, message(),
+        begin
+            Msg = emqx_message:from_map(
+                    inject_magic_into(from, emqx_message:to_map(Msg0))),
+            OutMsg= emqx_hooks:run_fold('message.publish', [], Msg),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true ->
+                    ?assertEqual(Msg, OutMsg),
+                    skip;
+                _ ->
+                    ExpectedOutMsg = case emqx_message:from(Msg) of
+                                         <<"baduser">> ->
+                                             MsgMap = emqx_message:to_map(Msg),
+                                             emqx_message:from_map(
+                                               MsgMap#{qos => 0,
+                                                       topic => <<"">>,
+                                                       payload => <<"">>
+                                                      });
+                                         <<"gooduser">> = From ->
+                                             MsgMap = emqx_message:to_map(Msg),
+                                             emqx_message:from_map(
+                                               MsgMap#{topic => From,
+                                                       payload => From
+                                                      });
+                                         _ -> Msg
+                                     end,
+                    ?assertEqual(ExpectedOutMsg, OutMsg),
+
+                    {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+        end).
+
+prop_message_dropped() ->
+    ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()},
+        begin
+            ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{reason => stringfy(Reason),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+       end).
+
+prop_message_delivered() ->
+    ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
+        begin
+            ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{clientinfo => from_clientinfo(ClientInfo),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+       end).
+
+prop_message_acked() ->
+    ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
+        begin
+            ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{clientinfo => from_clientinfo(ClientInfo),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+        end).
+
+nodestr() ->
+    stringfy(node()).
+
+peerhost(#{peername := {Host, _}}) ->
+    ntoa(Host).
+
+sockport(#{sockname := {_, Port}}) ->
+    Port.
+
+%% copied from emqx_exhook
+
+ntoa({0,0,0,0,0,16#ffff,AB,CD}) ->
+    list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}));
+ntoa(IP) ->
+    list_to_binary(inet_parse:ntoa(IP)).
+
+maybe(undefined) -> <<>>;
+maybe(B) -> B.
+
+properties(undefined) -> [];
+properties(M) when is_map(M) ->
+    maps:fold(fun(K, V, Acc) ->
+        [#{name => stringfy(K),
+           value => stringfy(V)} | Acc]
+    end, [], M).
+
+topicfilters(Tfs) when is_list(Tfs) ->
+    [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs].
+
+%% @private
+stringfy(Term) when is_binary(Term) ->
+    Term;
+stringfy(Term) when is_integer(Term) ->
+    integer_to_binary(Term);
+stringfy(Term) when is_atom(Term) ->
+    atom_to_binary(Term, utf8);
+stringfy(Term) ->
+    unicode:characters_to_binary((io_lib:format("~0p", [Term]))).
+
+subopts(SubOpts) ->
+    #{qos => maps:get(qos, SubOpts, 0),
+      rh => maps:get(rh, SubOpts, 0),
+      rap => maps:get(rap, SubOpts, 0),
+      nl => maps:get(nl, SubOpts, 0),
+      share => maps:get(share, SubOpts, <<>>)
+     }.
+
+authresult_to_bool(AuthResult) ->
+    maps:get(auth_result, AuthResult, undefined) == success.
+
+aclresult_to_bool(Result) ->
+    Result == allow.
+
+pubsub_to_enum(publish) -> 'PUBLISH';
+pubsub_to_enum(subscribe) -> 'SUBSCRIBE'.
+
+from_conninfo(ConnInfo) ->
+    #{node => nodestr(),
+      clientid => maps:get(clientid, ConnInfo),
+      username => maybe(maps:get(username, ConnInfo, <<>>)),
+      peerhost => peerhost(ConnInfo),
+      sockport => sockport(ConnInfo),
+      proto_name => maps:get(proto_name, ConnInfo),
+      proto_ver => stringfy(maps:get(proto_ver, ConnInfo)),
+      keepalive => maps:get(keepalive, ConnInfo)
+     }.
+
+from_clientinfo(ClientInfo) ->
+    #{node => nodestr(),
+      clientid => maps:get(clientid, ClientInfo),
+      username => maybe(maps:get(username, ClientInfo, <<>>)),
+      password => maybe(maps:get(password, ClientInfo, <<>>)),
+      peerhost => ntoa(maps:get(peerhost, ClientInfo)),
+      sockport => maps:get(sockport, ClientInfo),
+      protocol => stringfy(maps:get(protocol, ClientInfo)),
+      mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)),
+      is_superuser => maps:get(is_superuser, ClientInfo, false),
+      anonymous => maps:get(anonymous, ClientInfo, true),
+      cn => maybe(maps:get(cn, ClientInfo, <<>>)),
+      dn => maybe(maps:get(dn, ClientInfo, <<>>))
+    }.
+
+from_message(Msg) ->
+    #{node => nodestr(),
+      id => emqx_guid:to_hexstr(emqx_message:id(Msg)),
+      qos => emqx_message:qos(Msg),
+      from => stringfy(emqx_message:from(Msg)),
+      topic => emqx_message:topic(Msg),
+      payload => emqx_message:payload(Msg),
+      timestamp => emqx_message:timestamp(Msg)
+     }.
+
+%%--------------------------------------------------------------------
+%% Helper
+%%--------------------------------------------------------------------
+
+do_setup() ->
+    logger:set_primary_config(#{level => warning}),
+    _ = emqx_exhook_demo_svr:start(),
+    emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
+    %% waiting first loaded event
+    {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(),
+    ok.
+
+do_teardown(_) ->
+    emqx_ct_helpers:stop_apps([emqx_exhook]),
+    %% waiting last unloaded event
+    {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(),
+    _ = emqx_exhook_demo_svr:stop(),
+    logger:set_primary_config(#{level => notice}),
+    timer:sleep(2000),
+    ok.
+
+set_special_cfgs(emqx) ->
+    application:set_env(emqx, allow_anonymous, false),
+    application:set_env(emqx, enable_acl_cache, false),
+    application:set_env(emqx, modules_loaded_file, undefined),
+    application:set_env(emqx, plugins_loaded_file,
+                        emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins"));
+set_special_cfgs(emqx_exhook) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% Generators
+%%--------------------------------------------------------------------
+
+conn_properties() ->
+    #{}.
+
+ack_properties() ->
+    #{}.
+
+sub_properties() ->
+    #{}.
+
+unsub_properties() ->
+    #{}.
+
+shutdown_reason() ->
+    oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]).
+
+authresult() ->
+    ?LET(RC, connack_return_code(), #{auth_result => RC}).
+
+inject_magic_into(Key, Object) ->
+    case castspell() of
+        muggles -> Object;
+        Spell ->
+            Object#{Key => Spell}
+    end.
+
+castspell() ->
+    L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles],
+    lists:nth(rand:uniform(length(L)), L).
diff --git a/apps/emqx_gateway/etc/emqx_exhook.conf b/apps/emqx_gateway/etc/emqx_exhook.conf
deleted file mode 100644
index b2758e705..000000000
--- a/apps/emqx_gateway/etc/emqx_exhook.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-##====================================================================
-## EMQ X Hooks
-##====================================================================
-
-##--------------------------------------------------------------------
-## Server Address
-
-## The gRPC server url
-##
-## exhook.server.$name.url = url()
-exhook.server.default.url = "http://127.0.0.1:9000"
-
-#exhook.server.default.ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
-#exhook.server.default.ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
-#exhook.server.default.ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
diff --git a/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl b/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl
deleted file mode 100644
index cb2ab8d11..000000000
--- a/apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl
+++ /dev/null
@@ -1,531 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(prop_exhook_hooks).
-
-% -include_lib("proper/include/proper.hrl").
-% -include_lib("eunit/include/eunit.hrl").
-
-% -import(emqx_ct_proper_types,
-%         [ conninfo/0
-%         , clientinfo/0
-%         , sessioninfo/0
-%         , message/0
-%         , connack_return_code/0
-%         , topictab/0
-%         , topic/0
-%         , subopts/0
-%         ]).
-
-% -define(ALL(Vars, Types, Exprs),
-%         ?SETUP(fun() ->
-%             State = do_setup(),
-%             fun() -> do_teardown(State) end
-%          end, ?FORALL(Vars, Types, Exprs))).
-
-% %%--------------------------------------------------------------------
-% %% Properties
-% %%--------------------------------------------------------------------
-
-% prop_client_connect() ->
-%     ?ALL({ConnInfo, ConnProps},
-%          {conninfo(), conn_properties()},
-%        begin
-%            ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]),
-%            {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(),
-%            Expected =
-%                #{props => properties(ConnProps),
-%                  conninfo => from_conninfo(ConnInfo)
-%                 },
-%            ?assertEqual(Expected, Resp),
-%            true
-%        end).
-
-% prop_client_connack() ->
-%     ?ALL({ConnInfo, Rc, AckProps},
-%          {conninfo(), connack_return_code(), ack_properties()},
-%         begin
-%             ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]),
-%             {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(AckProps),
-%                   result_code => atom_to_binary(Rc, utf8),
-%                   conninfo => from_conninfo(ConnInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_authenticate() ->
-%     ?ALL({ClientInfo0, AuthResult},
-%          {clientinfo(), authresult()},
-%         begin
-%             ClientInfo = inject_magic_into(username, ClientInfo0),
-%             OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult),
-%             ExpectedAuthResult = case maps:get(username, ClientInfo) of
-%                                      <<"baduser">> ->
-%                                          AuthResult#{
-%                                            auth_result => not_authorized,
-%                                            anonymous => false};
-%                                      <<"gooduser">> ->
-%                                          AuthResult#{
-%                                            auth_result => success,
-%                                            anonymous => false};
-%                                      <<"normaluser">> ->
-%                                          AuthResult#{
-%                                            auth_result => success,
-%                                            anonymous => false};
-%                                      _ ->
-%                                          case maps:get(auth_result, AuthResult) of
-%                                              success ->
-%                                                  #{auth_result => success,
-%                                                    anonymous => false};
-%                                              _ ->
-%                                                  #{auth_result => not_authorized,
-%                                                    anonymous => false}
-%                                          end
-%                                  end,
-%             ?assertEqual(ExpectedAuthResult, OutAuthResult),
-
-%             {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{result => authresult_to_bool(AuthResult),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_authorize() ->
-%     ?ALL({ClientInfo0, PubSub, Topic, Result},
-%          {clientinfo(), oneof([publish, subscribe]),
-%           topic(), oneof([allow, deny])},
-%         begin
-%             ClientInfo = inject_magic_into(username, ClientInfo0),
-%             OutResult = emqx_hooks:run_fold(
-%                           'client.authorize',
-%                           [ClientInfo, PubSub, Topic],
-%                           Result),
-%             ExpectedOutResult = case maps:get(username, ClientInfo) of
-%                                     <<"baduser">> -> deny;
-%                                     <<"gooduser">> -> allow;
-%                                     <<"normaluser">> -> allow;
-%                                     _ -> Result
-%                                  end,
-%             ?assertEqual(ExpectedOutResult, OutResult),
-
-%             {'on_client_authorize', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{result => authzresult_to_bool(Result),
-%                   type => pubsub_to_enum(PubSub),
-%                   topic => Topic,
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_connected() ->
-%     ?ALL({ClientInfo, ConnInfo},
-%          {clientinfo(), conninfo()},
-%         begin
-%             ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]),
-%             {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_disconnected() ->
-%     ?ALL({ClientInfo, Reason, ConnInfo},
-%          {clientinfo(), shutdown_reason(), conninfo()},
-%         begin
-%             ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]),
-%             {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{reason => stringfy(Reason),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_subscribe() ->
-%     ?ALL({ClientInfo, SubProps, TopicTab},
-%          {clientinfo(), sub_properties(), topictab()},
-%         begin
-%             ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]),
-%             {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(SubProps),
-%                   topic_filters => topicfilters(TopicTab),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_unsubscribe() ->
-%     ?ALL({ClientInfo, UnSubProps, TopicTab},
-%          {clientinfo(), unsub_properties(), topictab()},
-%         begin
-%             ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]),
-%             {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(UnSubProps),
-%                   topic_filters => topicfilters(TopicTab),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_created() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]),
-%             {'on_session_created', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%              ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_subscribed() ->
-%     ?ALL({ClientInfo, Topic, SubOpts},
-%          {clientinfo(), topic(), subopts()},
-%         begin
-%             ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]),
-%             {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{topic => Topic,
-%                   subopts => subopts(SubOpts),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_unsubscribed() ->
-%     ?ALL({ClientInfo, Topic, SubOpts},
-%          {clientinfo(), topic(), subopts()},
-%         begin
-%             ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]),
-%             {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{topic => Topic,
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_resumed() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]),
-%             {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_discared() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]),
-%             {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_takeovered() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]),
-%             {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_terminated() ->
-%     ?ALL({ClientInfo, Reason, SessInfo},
-%          {clientinfo(), shutdown_reason(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]),
-%             {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{reason => stringfy(Reason),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_message_publish() ->
-%     ?ALL(Msg0, message(),
-%         begin
-%             Msg = emqx_message:from_map(
-%                     inject_magic_into(from, emqx_message:to_map(Msg0))),
-%             OutMsg= emqx_hooks:run_fold('message.publish', [], Msg),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true ->
-%                     ?assertEqual(Msg, OutMsg),
-%                     skip;
-%                 _ ->
-%                     ExpectedOutMsg = case emqx_message:from(Msg) of
-%                                          <<"baduser">> ->
-%                                              MsgMap = emqx_message:to_map(Msg),
-%                                              emqx_message:from_map(
-%                                                MsgMap#{qos => 0,
-%                                                        topic => <<"">>,
-%                                                        payload => <<"">>
-%                                                       });
-%                                          <<"gooduser">> = From ->
-%                                              MsgMap = emqx_message:to_map(Msg),
-%                                              emqx_message:from_map(
-%                                                MsgMap#{topic => From,
-%                                                        payload => From
-%                                                       });
-%                                          _ -> Msg
-%                                      end,
-%                     ?assertEqual(ExpectedOutMsg, OutMsg),
-
-%                     {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%         end).
-
-% prop_message_dropped() ->
-%     ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()},
-%         begin
-%             ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{reason => stringfy(Reason),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%        end).
-
-% prop_message_delivered() ->
-%     ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
-%         begin
-%             ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{clientinfo => from_clientinfo(ClientInfo),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%        end).
-
-% prop_message_acked() ->
-%     ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
-%         begin
-%             ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{clientinfo => from_clientinfo(ClientInfo),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%         end).
-
-% nodestr() ->
-%     stringfy(node()).
-
-% peerhost(#{peername := {Host, _}}) ->
-%     ntoa(Host).
-
-% sockport(#{sockname := {_, Port}}) ->
-%     Port.
-
-% %% copied from emqx_exhook
-
-% ntoa({0,0,0,0,0,16#ffff,AB,CD}) ->
-%     list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}));
-% ntoa(IP) ->
-%     list_to_binary(inet_parse:ntoa(IP)).
-
-% maybe(undefined) -> <<>>;
-% maybe(B) -> B.
-
-% properties(undefined) -> [];
-% properties(M) when is_map(M) ->
-%     maps:fold(fun(K, V, Acc) ->
-%         [#{name => stringfy(K),
-%            value => stringfy(V)} | Acc]
-%     end, [], M).
-
-% topicfilters(Tfs) when is_list(Tfs) ->
-%     [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs].
-
-% %% @private
-% stringfy(Term) when is_binary(Term) ->
-%     Term;
-% stringfy(Term) when is_integer(Term) ->
-%     integer_to_binary(Term);
-% stringfy(Term) when is_atom(Term) ->
-%     atom_to_binary(Term, utf8);
-% stringfy(Term) ->
-%     unicode:characters_to_binary((io_lib:format("~0p", [Term]))).
-
-% subopts(SubOpts) ->
-%     #{qos => maps:get(qos, SubOpts, 0),
-%       rh => maps:get(rh, SubOpts, 0),
-%       rap => maps:get(rap, SubOpts, 0),
-%       nl => maps:get(nl, SubOpts, 0),
-%       share => maps:get(share, SubOpts, <<>>)
-%      }.
-
-% authresult_to_bool(AuthResult) ->
-%     maps:get(auth_result, AuthResult, undefined) == success.
-
-% authzresult_to_bool(Result) ->
-%     Result == allow.
-
-% pubsub_to_enum(publish) -> 'PUBLISH';
-% pubsub_to_enum(subscribe) -> 'SUBSCRIBE'.
-
-% from_conninfo(ConnInfo) ->
-%     #{node => nodestr(),
-%       clientid => maps:get(clientid, ConnInfo),
-%       username => maybe(maps:get(username, ConnInfo, <<>>)),
-%       peerhost => peerhost(ConnInfo),
-%       sockport => sockport(ConnInfo),
-%       proto_name => maps:get(proto_name, ConnInfo),
-%       proto_ver => stringfy(maps:get(proto_ver, ConnInfo)),
-%       keepalive => maps:get(keepalive, ConnInfo)
-%      }.
-
-% from_clientinfo(ClientInfo) ->
-%     #{node => nodestr(),
-%       clientid => maps:get(clientid, ClientInfo),
-%       username => maybe(maps:get(username, ClientInfo, <<>>)),
-%       password => maybe(maps:get(password, ClientInfo, <<>>)),
-%       peerhost => ntoa(maps:get(peerhost, ClientInfo)),
-%       sockport => maps:get(sockport, ClientInfo),
-%       protocol => stringfy(maps:get(protocol, ClientInfo)),
-%       mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)),
-%       is_superuser => maps:get(is_superuser, ClientInfo, false),
-%       anonymous => maps:get(anonymous, ClientInfo, true),
-%       cn => maybe(maps:get(cn, ClientInfo, <<>>)),
-%       dn => maybe(maps:get(dn, ClientInfo, <<>>))
-%     }.
-
-% from_message(Msg) ->
-%     #{node => nodestr(),
-%       id => emqx_guid:to_hexstr(emqx_message:id(Msg)),
-%       qos => emqx_message:qos(Msg),
-%       from => stringfy(emqx_message:from(Msg)),
-%       topic => emqx_message:topic(Msg),
-%       payload => emqx_message:payload(Msg),
-%       timestamp => emqx_message:timestamp(Msg)
-%      }.
-
-% %%--------------------------------------------------------------------
-% %% Helper
-% %%--------------------------------------------------------------------
-
-% do_setup() ->
-%     logger:set_primary_config(#{level => warning}),
-%     _ = emqx_exhook_demo_svr:start(),
-%     emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
-%     %% waiting first loaded event
-%     {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(),
-%     ok.
-
-% do_teardown(_) ->
-%     emqx_ct_helpers:stop_apps([emqx_exhook]),
-%     %% waiting last unloaded event
-%     {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(),
-%     _ = emqx_exhook_demo_svr:stop(),
-%     logger:set_primary_config(#{level => notice}),
-%     timer:sleep(2000),
-%     ok.
-
-% set_special_cfgs(emqx) ->
-%     application:set_env(emqx, allow_anonymous, false),
-%     application:set_env(emqx, enable_authz_cache, false),
-%     application:set_env(emqx, modules_loaded_file, undefined),
-%     application:set_env(emqx, plugins_loaded_file,
-%                         emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins"));
-% set_special_cfgs(emqx_exhook) ->
-%     ok.
-
-% %%--------------------------------------------------------------------
-% %% Generators
-% %%--------------------------------------------------------------------
-
-% conn_properties() ->
-%     #{}.
-
-% ack_properties() ->
-%     #{}.
-
-% sub_properties() ->
-%     #{}.
-
-% unsub_properties() ->
-%     #{}.
-
-% shutdown_reason() ->
-%     oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]).
-
-% authresult() ->
-%     ?LET(RC, connack_return_code(), #{auth_result => RC}).
-
-% inject_magic_into(Key, Object) ->
-%     case castspell() of
-%         muggles -> Object;
-%         Spell ->
-%             Object#{Key => Spell}
-%     end.
-
-% castspell() ->
-%     L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles],
-%     lists:nth(rand:uniform(length(L)), L).
diff --git a/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl
deleted file mode 100644
index 5dc29e6f1..000000000
--- a/apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl
+++ /dev/null
@@ -1,97 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_exhook_SUITE).
-
-% -compile(export_all).
-% -compile(nowarn_export_all).
-
-
-% -include_lib("eunit/include/eunit.hrl").
-% -include_lib("common_test/include/ct.hrl").
-
-% %%--------------------------------------------------------------------
-% %% Setups
-% %%--------------------------------------------------------------------
-
-% all() -> emqx_ct:all(?MODULE).
-
-% init_per_suite(Cfg) ->
-%     _ = emqx_exhook_demo_svr:start(),
-%     emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
-%     Cfg.
-
-% end_per_suite(_Cfg) ->
-%     emqx_ct_helpers:stop_apps([emqx_exhook]),
-%     emqx_exhook_demo_svr:stop().
-
-% set_special_cfgs(emqx) ->
-%     application:set_env(emqx, allow_anonymous, false),
-%     application:set_env(emqx, enable_authz_cache, false),
-%     application:set_env(emqx, plugins_loaded_file, undefined),
-%     application:set_env(emqx, modules_loaded_file, undefined);
-% set_special_cfgs(emqx_exhook) ->
-%     ok.
-
-% %%--------------------------------------------------------------------
-% %% Test cases
-% %%--------------------------------------------------------------------
-
-% t_noserver_nohook(_) ->
-%     emqx_exhook:disable(default),
-%     ?assertEqual([], ets:tab2list(emqx_hooks)),
-
-%     Opts = proplists:get_value(
-%              default,
-%              application:get_env(emqx_exhook, servers, [])
-%             ),
-%     ok = emqx_exhook:enable(default, Opts),
-%     ?assertNotEqual([], ets:tab2list(emqx_hooks)).
-
-% t_cli_list(_) ->
-%     meck_print(),
-%     ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]]
-%                 , emqx_exhook_cli:cli(["server", "list"])
-%                 ),
-%     unmeck_print().
-
-% t_cli_enable_disable(_) ->
-%     meck_print(),
-%     ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
-%     ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
-%     ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])),
-
-%     ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
-%     ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
-%     unmeck_print().
-
-% t_cli_stats(_) ->
-%     meck_print(),
-%     _ = emqx_exhook_cli:cli(["server", "stats"]),
-%     _ = emqx_exhook_cli:cli(x),
-%     unmeck_print().
-
-% %%--------------------------------------------------------------------
-% %% Utils
-% %%--------------------------------------------------------------------
-
-% meck_print() ->
-%     meck:new(emqx_ctl, [passthrough, no_history, no_link]),
-%     meck:expect(emqx_ctl, print, fun(_) -> ok end),
-%     meck:expect(emqx_ctl, print, fun(_, Args) -> Args end).
-
-% unmeck_print() ->
-%     meck:unload(emqx_ctl).
diff --git a/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl
deleted file mode 100644
index bcc8865ab..000000000
--- a/apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl
+++ /dev/null
@@ -1,339 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_exhook_demo_svr).
-
-% -behavior(emqx_exhook_v_1_hook_provider_bhvr).
-
-% %%
-% -export([ start/0
-%         , stop/0
-%         , take/0
-%         , in/1
-%         ]).
-
-% %% gRPC server HookProvider callbacks
-% -export([ on_provider_loaded/2
-%         , on_provider_unloaded/2
-%         , on_client_connect/2
-%         , on_client_connack/2
-%         , on_client_connected/2
-%         , on_client_disconnected/2
-%         , on_client_authenticate/2
-%         , on_client_authorize/2
-%         , on_client_subscribe/2
-%         , on_client_unsubscribe/2
-%         , on_session_created/2
-%         , on_session_subscribed/2
-%         , on_session_unsubscribed/2
-%         , on_session_resumed/2
-%         , on_session_discarded/2
-%         , on_session_takeovered/2
-%         , on_session_terminated/2
-%         , on_message_publish/2
-%         , on_message_delivered/2
-%         , on_message_dropped/2
-%         , on_message_acked/2
-%         ]).
-
-% -define(PORT, 9000).
-% -define(NAME, ?MODULE).
-
-% %%--------------------------------------------------------------------
-% %% Server APIs
-% %%--------------------------------------------------------------------
-
-% start() ->
-%     Pid = spawn(fun mngr_main/0),
-%     register(?MODULE, Pid),
-%     {ok, Pid}.
-
-% stop() ->
-%     grpc:stop_server(?NAME),
-%     ?MODULE ! stop.
-
-% take() ->
-%     ?MODULE ! {take, self()},
-%     receive {value, V} -> V
-%     after 5000 -> error(timeout) end.
-
-% in({FunName, Req}) ->
-%     ?MODULE ! {in, FunName, Req}.
-
-% mngr_main() ->
-%     application:ensure_all_started(grpc),
-%     Services = #{protos => [emqx_exhook_pb],
-%                  services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
-%                 },
-%     Options = [],
-%     Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
-%     mngr_loop([Svr, queue:new(), queue:new()]).
-
-% mngr_loop([Svr, Q, Takes]) ->
-%     receive
-%         {in, FunName, Req} ->
-%             {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
-%             mngr_loop([Svr, NQ1, NQ2]);
-%         {take, From} ->
-%             {NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
-%             mngr_loop([Svr, NQ1, NQ2]);
-%         stop ->
-%             exit(normal)
-%     end.
-
-% reply(Q1, Q2) ->
-%     case queue:len(Q1) =:= 0 orelse
-%          queue:len(Q2) =:= 0 of
-%         true -> {Q1, Q2};
-%         _ ->
-%             {{value, {Name, V}}, NQ1} = queue:out(Q1),
-%             {{value, From}, NQ2} = queue:out(Q2),
-%             From ! {value, {Name, V}},
-%             {NQ1, NQ2}
-%     end.
-
-% %%--------------------------------------------------------------------
-% %% callbacks
-% %%--------------------------------------------------------------------
-
-% -spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-
-% on_provider_loaded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{hooks => [
-%                      #{name => <<"client.connect">>},
-%                      #{name => <<"client.connack">>},
-%                      #{name => <<"client.connected">>},
-%                      #{name => <<"client.disconnected">>},
-%                      #{name => <<"client.authenticate">>},
-%                      #{name => <<"client.authorize">>},
-%                      #{name => <<"client.subscribe">>},
-%                      #{name => <<"client.unsubscribe">>},
-%                      #{name => <<"session.created">>},
-%                      #{name => <<"session.subscribed">>},
-%                      #{name => <<"session.unsubscribed">>},
-%                      #{name => <<"session.resumed">>},
-%                      #{name => <<"session.discarded">>},
-%                      #{name => <<"session.takeovered">>},
-%                      #{name => <<"session.terminated">>},
-%                      #{name => <<"message.publish">>},
-%                      #{name => <<"message.delivered">>},
-%                      #{name => <<"message.acked">>},
-%                      #{name => <<"message.dropped">>}]}, Md}.
-% -spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_provider_unloaded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connect(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connack(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connected(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_disconnected(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case Username of
-%         <<"baduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, false}}, Md};
-%         <<"gooduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, true}}, Md};
-%         <<"normaluser">> ->
-%             {ok, #{type => 'CONTINUE',
-%                    value => {bool_result, true}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_client_authorize(emqx_exhook_pb:client_authorize_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_authorize(#{clientinfo := #{username := Username}} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case Username of
-%         <<"baduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, false}}, Md};
-%         <<"gooduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, true}}, Md};
-%         <<"normaluser">> ->
-%             {ok, #{type => 'CONTINUE',
-%                    value => {bool_result, true}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_subscribe(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_unsubscribe(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_created(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_subscribed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_unsubscribed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_resumed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_discarded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_takeovered(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_terminated(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_publish(#{message := #{from := From} = Msg} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case From of
-%         <<"baduser">> ->
-%             NMsg = Msg#{qos => 0,
-%                         topic => <<"">>,
-%                         payload => <<"">>
-%                        },
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {message, NMsg}}, Md};
-%         <<"gooduser">> ->
-%             NMsg = Msg#{topic => From,
-%                         payload => From},
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {message, NMsg}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_delivered(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_dropped(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_acked(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
diff --git a/rebar.config.erl b/rebar.config.erl
index 5eca3094a..4098bb7ab 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -275,6 +275,7 @@ relx_apps(ReleaseType) ->
     , emqx_authn
     , emqx_authz
     , emqx_gateway
+    , {emqx_exhook, load}
     , emqx_data_bridge
     , emqx_rule_engine
     , emqx_rule_actions

From aa019b34ad8e975c3796ebed712777ea109d1680 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Sat, 31 Jul 2021 12:11:34 +0800
Subject: [PATCH 17/27] test(exhook): refine tests

---
 apps/emqx_exhook/include/emqx_exhook.hrl      |  2 +-
 apps/emqx_exhook/priv/emqx_exhook.schema      | 38 ------------------
 apps/emqx_exhook/rebar.config                 |  8 ----
 apps/emqx_exhook/src/emqx_exhook_cli.erl      |  2 +-
 apps/emqx_exhook/src/emqx_exhook_handler.erl  |  7 ++--
 apps/emqx_exhook/src/emqx_exhook_server.erl   |  1 -
 apps/emqx_exhook/test/emqx_exhook_SUITE.erl   | 39 +++++++++++--------
 .../emqx_exhook/test/emqx_exhook_demo_svr.erl |  8 ++--
 scripts/find-apps.sh                          |  2 +-
 9 files changed, 33 insertions(+), 74 deletions(-)
 delete mode 100644 apps/emqx_exhook/priv/emqx_exhook.schema

diff --git a/apps/emqx_exhook/include/emqx_exhook.hrl b/apps/emqx_exhook/include/emqx_exhook.hrl
index 7301fdcbb..64131735e 100644
--- a/apps/emqx_exhook/include/emqx_exhook.hrl
+++ b/apps/emqx_exhook/include/emqx_exhook.hrl
@@ -25,7 +25,7 @@
       , {'client.connected',    {emqx_exhook_handler, on_client_connected,     []}}
       , {'client.disconnected', {emqx_exhook_handler, on_client_disconnected,  []}}
       , {'client.authenticate', {emqx_exhook_handler, on_client_authenticate,  []}}
-      , {'client.check_acl',    {emqx_exhook_handler, on_client_check_acl,     []}}
+      , {'client.authorize',    {emqx_exhook_handler, on_client_authorize,     []}}
       , {'client.subscribe',    {emqx_exhook_handler, on_client_subscribe,     []}}
       , {'client.unsubscribe',  {emqx_exhook_handler, on_client_unsubscribe,   []}}
       , {'session.created',     {emqx_exhook_handler, on_session_created,      []}}
diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema
deleted file mode 100644
index e5481a3dd..000000000
--- a/apps/emqx_exhook/priv/emqx_exhook.schema
+++ /dev/null
@@ -1,38 +0,0 @@
-%%-*- mode: erlang -*-
-
-{mapping, "exhook.server.$name.url", "emqx_exhook.servers", [
-  {datatype, string}
-]}.
-
-{mapping, "exhook.server.$name.ssl.cacertfile", "emqx_exhook.servers", [
-  {datatype, string}
-]}.
-
-{mapping, "exhook.server.$name.ssl.certfile", "emqx_exhook.servers", [
-  {datatype, string}
-]}.
-
-{mapping, "exhook.server.$name.ssl.keyfile", "emqx_exhook.servers", [
-  {datatype, string}
-]}.
-
-{translation, "emqx_exhook.servers", fun(Conf) ->
-  Filter  = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
-  ServerOptions = fun(Prefix) ->
-                      case http_uri:parse(cuttlefish:conf_get(Prefix ++ ".url", Conf)) of
-                          {ok, {http, _, Host, Port, _, _}} ->
-                              [{scheme, http}, {host, Host}, {port, Port}];
-                          {ok, {https, _, Host, Port, _, _}} ->
-                              [{scheme, https}, {host, Host}, {port, Port},
-                               {ssl_options,
-                                 Filter([{ssl, true},
-                                         {certfile, cuttlefish:conf_get(Prefix ++ ".ssl.certfile", Conf, undefined)},
-                                         {keyfile, cuttlefish:conf_get(Prefix ++ ".ssl.keyfile", Conf, undefined)},
-                                         {cacertfile, cuttlefish:conf_get(Prefix ++ ".ssl.cacertfile", Conf, undefined)}
-                                        ])}];
-                          _ -> error(invalid_server_options)
-                      end
-                  end,
-  [{list_to_atom(Name), ServerOptions("exhook.server." ++ Name)}
-   || {["exhook", "server", Name, "url"], _} <- cuttlefish_variable:filter_by_prefix("exhook.server", Conf)]
-end}.
diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config
index 883aad9bd..89dcb20a7 100644
--- a/apps/emqx_exhook/rebar.config
+++ b/apps/emqx_exhook/rebar.config
@@ -39,11 +39,3 @@
 {cover_excl_mods, [emqx_exhook_pb,
                    emqx_exhook_v_1_hook_provider_bhvr,
                    emqx_exhook_v_1_hook_provider_client]}.
-
-{profiles,
- [{test,
-   [{deps,
-      [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}}
-      ]}
-    ]}
-]}.
diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl
index a8dc43b16..3f0dd0b6c 100644
--- a/apps/emqx_exhook/src/emqx_exhook_cli.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_cli.erl
@@ -30,7 +30,7 @@ cli(["server", "list"]) ->
 cli(["server", "enable", Name0]) ->
     if_enabled(fun() ->
         Name = list_to_atom(Name0),
-        case proplists:get_value(Name, application:get_env(?APP, servers, [])) of
+        case maps:get(Name, emqx_config:get([exhook, server]), undefined) of
             undefined ->
                 emqx_ctl:print("not_found~n");
             Opts ->
diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl
index d2fb84bab..1e81646e0 100644
--- a/apps/emqx_exhook/src/emqx_exhook_handler.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl
@@ -87,7 +87,6 @@ on_client_disconnected(ClientInfo, Reason, _ConnInfo) ->
            },
     cast('client.disconnected', Req).
 
-%% FIXME: `AuthResult`
 on_client_authenticate(ClientInfo, AuthResult) ->
     %% XXX: Bool is missing more information about the atom of the result
     %%      So, the `Req` has missed detailed info too.
@@ -95,7 +94,7 @@ on_client_authenticate(ClientInfo, AuthResult) ->
     %%      The return value of `call_fold` just a bool, that has missed
     %%      detailed info too.
     %%
-    Bool = maps:get(auth_result, AuthResult, undefined) == success,
+    Bool = AuthResult == ok,
     Req = #{clientinfo => clientinfo(ClientInfo),
             result => Bool
            },
@@ -103,8 +102,8 @@ on_client_authenticate(ClientInfo, AuthResult) ->
     case call_fold('client.authenticate', Req,
                    fun merge_responsed_bool/2) of
         {StopOrOk, #{result := Result0}} when is_boolean(Result0) ->
-            Result = case Result0 of true -> success; _ -> not_authorized end,
-            {StopOrOk, AuthResult#{auth_result => Result, anonymous => false}};
+            Result = case Result0 of true -> ok; _ -> {error, not_authorized} end,
+            {StopOrOk, Result};
         _ ->
             {ok, AuthResult}
     end.
diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl
index 5bda76d13..674bfea5a 100644
--- a/apps/emqx_exhook/src/emqx_exhook_server.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_server.erl
@@ -122,7 +122,6 @@ to_list(Name) when is_list(Name) ->
 
 %% @private
 channel_opts(Opts = #{url := URL}) ->
-    io:format("~p~n", [Opts]),
     case uri_string:parse(URL) of
         #{scheme := <<"http">>, host := Host, port := Port} ->
             {format_http_uri("http", Host, Port), #{}};
diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
index 5d5a396a5..0291a8f91 100644
--- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
@@ -22,6 +22,10 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
+-define(CONF_DEFAULT, <<"
+exhook: { server.default: { url: \"http://127.0.0.1:9000\" } }
+">>).
+
 %%--------------------------------------------------------------------
 %% Setups
 %%--------------------------------------------------------------------
@@ -30,35 +34,24 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
     _ = emqx_exhook_demo_svr:start(),
-    emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
+    ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
+    emqx_ct_helpers:start_apps([emqx_exhook]),
     Cfg.
 
 end_per_suite(_Cfg) ->
     emqx_ct_helpers:stop_apps([emqx_exhook]),
     emqx_exhook_demo_svr:stop().
 
-set_special_cfgs(emqx) ->
-    application:set_env(emqx, allow_anonymous, false),
-    application:set_env(emqx, enable_acl_cache, false),
-    application:set_env(emqx, plugins_loaded_file, undefined),
-    application:set_env(emqx, modules_loaded_file, undefined);
-set_special_cfgs(emqx_exhook) ->
-    ok.
-
 %%--------------------------------------------------------------------
 %% Test cases
 %%--------------------------------------------------------------------
 
 t_noserver_nohook(_) ->
     emqx_exhook:disable(default),
-    ?assertEqual([], ets:tab2list(emqx_hooks)),
-
-    Opts = proplists:get_value(
-             default,
-             application:get_env(emqx_exhook, servers, [])
-            ),
+    ?assertEqual([], loaded_exhook_hookpoints()),
+    Opts = emqx_config:get([exhook, server, default]),
     ok = emqx_exhook:enable(default, Opts),
-    ?assertNotEqual([], ets:tab2list(emqx_hooks)).
+    ?assertNotEqual([], loaded_exhook_hookpoints()).
 
 t_cli_list(_) ->
     meck_print(),
@@ -94,3 +87,17 @@ meck_print() ->
 
 unmeck_print() ->
     meck:unload(emqx_ctl).
+
+loaded_exhook_hookpoints() ->
+    lists:filtermap(fun(E) ->
+        Name = element(2, E),
+        Callbacks = element(3, E),
+        case lists:any(fun is_exhook_callback/1, Callbacks) of
+            true -> {true, Name};
+            _ -> false
+        end
+    end, ets:tab2list(emqx_hooks)).
+
+is_exhook_callback(Cb) ->
+    Action = element(2, Cb),
+    emqx_exhook_handler == element(1, Action).
diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
index c2db04dd4..656788b5e 100644
--- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
@@ -33,7 +33,7 @@
         , on_client_connected/2
         , on_client_disconnected/2
         , on_client_authenticate/2
-        , on_client_check_acl/2
+        , on_client_authorize/2
         , on_client_subscribe/2
         , on_client_unsubscribe/2
         , on_session_created/2
@@ -122,7 +122,7 @@ on_provider_loaded(Req, Md) ->
                      #{name => <<"client.connected">>},
                      #{name => <<"client.disconnected">>},
                      #{name => <<"client.authenticate">>},
-                     #{name => <<"client.check_acl">>},
+                     #{name => <<"client.authorize">>},
                      #{name => <<"client.subscribe">>},
                      #{name => <<"client.unsubscribe">>},
                      #{name => <<"session.created">>},
@@ -197,10 +197,10 @@ on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) ->
             {ok, #{type => 'IGNORE'}, Md}
     end.
 
--spec on_client_check_acl(emqx_exhook_pb:client_check_acl_request(), grpc:metadata())
+-spec on_client_authorize(emqx_exhook_pb:client_authorize_request(), grpc:metadata())
     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
      | {error, grpc_cowboy_h:error_response()}.
-on_client_check_acl(#{clientinfo := #{username := Username}} = Req, Md) ->
+on_client_authorize(#{clientinfo := #{username := Username}} = Req, Md) ->
     ?MODULE:in({?FUNCTION_NAME, Req}),
     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
     %% some cases for testing
diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh
index e437c49c5..177a4f57c 100755
--- a/scripts/find-apps.sh
+++ b/scripts/find-apps.sh
@@ -7,7 +7,7 @@ cd -P -- "$(dirname -- "$0")/.."
 
 find_app() {
     local appdir="$1"
-    find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_exhook|emqx_exproto|emqx_lwm2m|emqx_sn|emqx_coap|emqx_stomp|emqx_dashboard"
+    find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_dashboard"
 }
 
 find_app 'apps'

From 7c46c8712b5e7f1e86c314fc2627815b45ddb8fe Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Sat, 31 Jul 2021 12:15:01 +0800
Subject: [PATCH 18/27] docs(exhook): renanme CheckAcl to Authorize

---
 apps/emqx_exhook/docs/design-cn.md | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md
index 6686e96e3..423a53bf5 100644
--- a/apps/emqx_exhook/docs/design-cn.md
+++ b/apps/emqx_exhook/docs/design-cn.md
@@ -68,7 +68,7 @@ service HookProvider {
 
   rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {};
 
-  rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {};
+  rpc OnClientAuthorize(ClientAuthorizeRequest) returns (ValuedResponse) {};
 
   rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {};
 
@@ -101,16 +101,12 @@ service HookProvider {
 ### 配置文件示例
 
 ```
-## 配置 gRPC 服务地址 (HTTP)
-##
-## s1 为服务器的名称
-exhook.server.s1.url = http://127.0.0.1:9001
-
-## 配置 gRPC 服务地址 (HTTPS)
-##
-## s2 为服务器名称
-exhook.server.s2.url = https://127.0.0.1:9002
-exhook.server.s2.cacertfile = ca.pem
-exhook.server.s2.certfile = cert.pem
-exhook.server.s2.keyfile = key.pem
+exhook: {
+    ## 配置 gRPC 服务地址 (HTTP)
+    ##
+    ## default 为服务器的名称
+    server.default: {
+        url: "http://127.0.0.1:9000"
+    }
+}
 ```

From 922aa7aae5ead0793aafb3b7288253063900aeb8 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Sat, 31 Jul 2021 14:12:57 +0800
Subject: [PATCH 19/27] test(exhook): refine property-tests

---
 .../test/props/prop_exhook_hooks.erl          | 59 ++++++++-----------
 1 file changed, 23 insertions(+), 36 deletions(-)

diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
index 24f45c8b0..229805d3b 100644
--- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
+++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
@@ -30,12 +30,17 @@
         , subopts/0
         ]).
 
+-define(CONF_DEFAULT, <<"
+exhook: { server.default: { url: \"http://127.0.0.1:9000\" } }
+">>).
+
 -define(ALL(Vars, Types, Exprs),
         ?SETUP(fun() ->
             State = do_setup(),
             fun() -> do_teardown(State) end
          end, ?FORALL(Vars, Types, Exprs))).
 
+
 %%--------------------------------------------------------------------
 %% Properties
 %%--------------------------------------------------------------------
@@ -76,27 +81,13 @@ prop_client_authenticate() ->
             ClientInfo = inject_magic_into(username, ClientInfo0),
             OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult),
             ExpectedAuthResult = case maps:get(username, ClientInfo) of
-                                     <<"baduser">> ->
-                                         AuthResult#{
-                                           auth_result => not_authorized,
-                                           anonymous => false};
-                                     <<"gooduser">> ->
-                                         AuthResult#{
-                                           auth_result => success,
-                                           anonymous => false};
-                                     <<"normaluser">> ->
-                                         AuthResult#{
-                                           auth_result => success,
-                                           anonymous => false};
-                                     _ ->
-                                         case maps:get(auth_result, AuthResult) of
-                                             success ->
-                                                 #{auth_result => success,
-                                                   anonymous => false};
-                                             _ ->
-                                                 #{auth_result => not_authorized,
-                                                   anonymous => false}
-                                         end
+                                     <<"baduser">> -> {error, not_authorized};
+                                     <<"gooduser">> -> ok;
+                                     <<"normaluser">> -> ok;
+                                     _ -> case AuthResult of
+                                              ok -> ok;
+                                              _ -> {error, not_authorized}
+                                          end
                                  end,
             ?assertEqual(ExpectedAuthResult, OutAuthResult),
 
@@ -109,14 +100,14 @@ prop_client_authenticate() ->
             true
         end).
 
-prop_client_check_acl() ->
+prop_client_authorize() ->
     ?ALL({ClientInfo0, PubSub, Topic, Result},
          {clientinfo(), oneof([publish, subscribe]),
           topic(), oneof([allow, deny])},
         begin
             ClientInfo = inject_magic_into(username, ClientInfo0),
             OutResult = emqx_hooks:run_fold(
-                          'client.check_acl',
+                          'client.authorize',
                           [ClientInfo, PubSub, Topic],
                           Result),
             ExpectedOutResult = case maps:get(username, ClientInfo) of
@@ -127,7 +118,7 @@ prop_client_check_acl() ->
                                  end,
             ?assertEqual(ExpectedOutResult, OutResult),
 
-            {'on_client_check_acl', Resp} = emqx_exhook_demo_svr:take(),
+            {'on_client_authorize', Resp} = emqx_exhook_demo_svr:take(),
             Expected =
                 #{result => aclresult_to_bool(Result),
                   type => pubsub_to_enum(PubSub),
@@ -423,7 +414,7 @@ subopts(SubOpts) ->
      }.
 
 authresult_to_bool(AuthResult) ->
-    maps:get(auth_result, AuthResult, undefined) == success.
+    AuthResult == ok.
 
 aclresult_to_bool(Result) ->
     Result == allow.
@@ -474,7 +465,8 @@ from_message(Msg) ->
 do_setup() ->
     logger:set_primary_config(#{level => warning}),
     _ = emqx_exhook_demo_svr:start(),
-    emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
+    ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
+    emqx_ct_helpers:start_apps([emqx_exhook]),
     %% waiting first loaded event
     {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(),
     ok.
@@ -488,15 +480,6 @@ do_teardown(_) ->
     timer:sleep(2000),
     ok.
 
-set_special_cfgs(emqx) ->
-    application:set_env(emqx, allow_anonymous, false),
-    application:set_env(emqx, enable_acl_cache, false),
-    application:set_env(emqx, modules_loaded_file, undefined),
-    application:set_env(emqx, plugins_loaded_file,
-                        emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins"));
-set_special_cfgs(emqx_exhook) ->
-    ok.
-
 %%--------------------------------------------------------------------
 %% Generators
 %%--------------------------------------------------------------------
@@ -517,7 +500,11 @@ shutdown_reason() ->
     oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]).
 
 authresult() ->
-    ?LET(RC, connack_return_code(), #{auth_result => RC}).
+    ?LET(RC, connack_return_code(),
+         case RC of
+             success -> ok;
+             _ -> {error, RC}
+         end).
 
 inject_magic_into(Key, Object) ->
     case castspell() of

From 604e282978fbed7c51dae113a3f8a2e1f5eaa105 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Sat, 31 Jul 2021 14:40:03 +0800
Subject: [PATCH 20/27] chore(exhook): fix dialyzer warnings

---
 apps/emqx_exhook/src/emqx_exhook.erl        | 1 -
 apps/emqx_exhook/src/emqx_exhook_server.erl | 8 ++------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_exhook/src/emqx_exhook.erl b/apps/emqx_exhook/src/emqx_exhook.erl
index e993858cf..e016190e0 100644
--- a/apps/emqx_exhook/src/emqx_exhook.erl
+++ b/apps/emqx_exhook/src/emqx_exhook.erl
@@ -106,7 +106,6 @@ call_fold(Hookpoint, Req, AccFun, [ServiceName|More]) ->
 %%----------------------------------------------------------
 %% Storage
 
--compile({inline, [save/2]}).
 save(Name, ServiceState) ->
     Saved = persistent_term:get(?APP, []),
     persistent_term:put(?APP, lists:reverse([Name | Saved])),
diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl
index 674bfea5a..9e5f8ce11 100644
--- a/apps/emqx_exhook/src/emqx_exhook_server.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_server.erl
@@ -40,7 +40,7 @@
           %% Server name (equal to grpc client channel name)
           name :: server_name(),
           %% The server started options
-          options :: list(),
+          options :: options(),
           %% gRPC channel pid
           channel :: pid(),
           %% Registered hook names and options
@@ -142,11 +142,7 @@ channel_opts(Opts = #{url := URL}) ->
             error(bad_server_url)
     end.
 
-format_http_uri(Scheme, Host0, Port) ->
-    Host = case is_tuple(Host0) of
-               true -> inet:ntoa(Host0);
-               _ -> Host0
-           end,
+format_http_uri(Scheme, Host, Port) ->
     lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])).
 
 filter(Ls) ->

From d7322aff193edf0b73a8a03e2d5dd36bf5f1fddc Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Tue, 3 Aug 2021 09:37:29 +0800
Subject: [PATCH 21/27] chore(exhook): empty appup.src

---
 apps/emqx_exhook/src/emqx_exhook.appup.src | 24 ----------------------
 1 file changed, 24 deletions(-)

diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src
index 1da31dbbc..9e142d9e2 100644
--- a/apps/emqx_exhook/src/emqx_exhook.appup.src
+++ b/apps/emqx_exhook/src/emqx_exhook.appup.src
@@ -1,33 +1,9 @@
 %% -*-: erlang -*-
 {VSN,
  [
-    {"4.3.2", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.1", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.0", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
-    ]},
     {<<".*">>, []}
  ],
  [
-    {"4.3.2", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.1", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.0", [
-      {load_module, emqx_exhook_app, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []},
-      {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}
-    ]},
     {<<".*">>, []}
  ]
 }.

From 82321da7ad317682b8d51275f089f38c9ff7e24a Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Tue, 3 Aug 2021 09:56:38 +0800
Subject: [PATCH 22/27] chore(exhook): bump version

---
 apps/emqx_exhook/src/emqx_exhook.app.src | 2 +-
 apps/emqx_exhook/src/emqx_exhook_app.erl | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
index 39c408e20..cca972eeb 100644
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,6 +1,6 @@
 {application, emqx_exhook,
  [{description, "EMQ X Extension for Hook"},
-  {vsn, "4.3.3"},
+  {vsn, "5.0.0"},
   {modules, []},
   {registered, []},
   {mod, {emqx_exhook_app, []}},
diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl
index b057ce463..9cbb2e1b8 100644
--- a/apps/emqx_exhook/src/emqx_exhook_app.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_app.erl
@@ -20,8 +20,6 @@
 
 -include("emqx_exhook.hrl").
 
--emqx_plugin(extension).
-
 -define(CNTER, emqx_exhook_counter).
 
 -export([ start/2

From 32534d2372968e7d2551aba9e9030bcb2e5b070b Mon Sep 17 00:00:00 2001
From: Turtle <deng@emqx.io>
Date: Tue, 3 Aug 2021 16:48:19 +0800
Subject: [PATCH 23/27] chore(dashboard): update get-dashboard script

---
 Makefile                 |  2 +-
 scripts/get-dashboard.sh | 40 ++++------------------------------------
 2 files changed, 5 insertions(+), 37 deletions(-)

diff --git a/Makefile b/Makefile
index c39270367..c3c0cc309 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
+export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.3
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 endif
diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh
index c4468ce80..a633fdae2 100755
--- a/scripts/get-dashboard.sh
+++ b/scripts/get-dashboard.sh
@@ -6,21 +6,10 @@ set -euo pipefail
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 
 RELEASE_ASSET_FILE="emqx-dashboard.zip"
-
-if [ -f 'EMQX_ENTERPRISE' ]; then
-    VERSION="${EMQX_EE_DASHBOARD_VERSION}"
-    DASHBOARD_PATH='lib-ee/emqx_dashboard/priv'
-    DASHBOARD_REPO='emqx-enterprise-dashboard-frontend-src'
-    AUTH="Authorization: token $(cat scripts/git-token)"
-    # have to be resolved with auth and redirect
-    DIRECT_DOWNLOAD_URL=""
-else
-    VERSION="${EMQX_CE_DASHBOARD_VERSION}"
-    DASHBOARD_PATH='apps/emqx_dashboard/priv'
-    DASHBOARD_REPO='emqx-dashboard-frontend'
-    AUTH=""
-    DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/emqx-dashboard.zip"
-fi
+VERSION="${EMQX_DASHBOARD_VERSION}"
+DASHBOARD_PATH='apps/emqx_dashboard/priv'
+DASHBOARD_REPO='emqx-dashboard-web'
+DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}"
 
 case $(uname) in
     *Darwin*) SED="sed -E";;
@@ -35,27 +24,6 @@ if [ -d "$DASHBOARD_PATH/www" ] && [ "$(version)" = "$VERSION" ]; then
     exit 0
 fi
 
-find_url() {
-    # Get the download URL of our desired asset
-    release_url="https://api.github.com/repos/emqx/${DASHBOARD_REPO}/releases/tags/${VERSION}"
-    release_info="$(curl --silent --show-error --header "${AUTH}" --header "Accept: application/vnd.github.v3+json" "$release_url")"
-    if ! download_url="$(echo "$release_info" | jq --raw-output ".assets[] | select(.name==\"${RELEASE_ASSET_FILE}\").url" | tr -d '\n' | tr -d '\r')"; then
-        echo "failed to query $release_url"
-        echo "${release_info}"
-        exit 1
-    fi
-    # Get GitHub's S3 redirect URL
-    curl --silent --show-error \
-         --header "${AUTH}" \
-         --header "Accept: application/octet-stream" \
-         --write-out "%{redirect_url}" \
-         "$download_url"
-}
-
-if [ -z "$DIRECT_DOWNLOAD_URL" ]; then
-    DIRECT_DOWNLOAD_URL="$(find_url)"
-fi
-
 curl -L --silent --show-error \
      --header "Accept: application/octet-stream" \
      --output "${RELEASE_ASSET_FILE}" \

From b69059d442b06f358cb624f564e47ecfed94278c Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Mon, 2 Aug 2021 15:50:17 +0800
Subject: [PATCH 24/27] build: easier build docker

Signed-off-by: zhanghongtong <rory-z@outlook.com>
---
 .github/workflows/build_packages.yaml |  68 +++++++---
 .github/workflows/run_fvt_tests.yaml  |   8 +-
 Makefile                              |   9 +-
 build                                 |  17 +++
 deploy/docker/Dockerfile              |  10 +-
 docker.mk                             | 186 --------------------------
 6 files changed, 81 insertions(+), 217 deletions(-)
 delete mode 100644 docker.mk

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 141645043..202686ad9 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -216,6 +216,8 @@ jobs:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        erl_otp:
+          - 23.2.7.2-emqx-2
         arch:
           - amd64
           - arm64
@@ -295,7 +297,7 @@ jobs:
         done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        ERL_OTP: erl${{ matrix.erl_otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -336,17 +338,14 @@ jobs:
 
   docker:
     runs-on: ubuntu-20.04
-
     needs: prepare
 
     strategy:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
-          - [amd64, x86_64]
-          - [arm64v8, aarch64]
-          - [arm32v7, arm]
+        erl_otp:
+          - 23.2.7.2-emqx-2
 
     steps:
     - uses: actions/download-artifact@v2
@@ -355,22 +354,49 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
       with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        image: tonistiigi/binfmt:latest
+        platforms: all
+    - name: build emqx docker image
+      if: github.event_name != 'release'
+      env:
+        ERL_OTP: erl${{ matrix.erl_otp }}
+        PROFILE: ${{ matrix.profile }}
+      run: |
+        cd source
+        PKG_VSN="$(./pkg-vsn.sh)"
+        docker buildx build --no-cache \
+           --platform=linux/amd64,linux/arm64 \
+           --build-arg PKG_VSN=$PKG_VSN  \
+           --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine  \
+           --build-arg RUN_FROM=alpine:3.14 \
+           --build-arg EMQX_NAME=$PROFILE \
+           --tag emqx/$PROFILE:$PKG_VSN \
+           -f deploy/docker/Dockerfile .
+    - uses: docker/login-action@v1
+      if: github.event_name == 'release'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - name: build emqx docker image
+      if: github.event_name == 'release'
+      env:
+        ERL_OTP: erl${{ matrix.erl_otp }}
+        PROFILE: ${{ matrix.profile }}
+      run: |
+        cd source
+        PKG_VSN="$(./pkg-vsn.sh)"
+        docker buildx build --no-cache \
+           --platform=linux/amd64,linux/arm64 \
+           --build-arg PKG_VSN=$PKG_VSN  \
+           --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine  \
+           --build-arg RUN_FROM=alpine:3.14 \
+           --build-arg EMQX_NAME=$PROFILE \
+           --tag emqx/$PROFILE:$PKG_VSN \
+           -f deploy/docker/Dockerfile \
+           --push .
 
   delete-artifact:
     runs-on: ubuntu-20.04
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index c6b160304..108434cb5 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -24,14 +24,16 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
+              echo "PROFILE=emqx" >> $GITHUB_ENV
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make $PROFILE-docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -79,11 +81,13 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "PROFILE=emqx" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make $PROFILE-docker
         - name: install k3s
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
diff --git a/Makefile b/Makefile
index c3c0cc309..6f21a6147 100644
--- a/Makefile
+++ b/Makefile
@@ -153,7 +153,14 @@ run: $(PROFILE) quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
 
-include docker.mk
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
 
 conf-segs:
 	@scripts/merge-config.escript
diff --git a/build b/build
index 08e33b1c2..2b030a1de 100755
--- a/build
+++ b/build
@@ -120,6 +120,20 @@ make_zip() {
     (cd "${tard}" && zip -qr - emqx) > "${zipball}"
 }
 
+make_docker() {
+    ## Build Docker image
+    echo "DOCKER BUILD: Build Docker image."
+    echo "DOCKER BUILD: build version -> $PKG_VSN."
+    echo "DOCKER BUILD: docker repo -> emqx/$PROFILE "
+
+    docker build --no-cache \
+       --build-arg PKG_VSN="$PKG_VSN"  \
+       --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine \
+       --build-arg EMQX_NAME="$PROFILE" \
+       --tag "emqx/$PROFILE:$PKG_VSN" \
+       -f deploy/docker/Dockerfile .
+}
+
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
@@ -140,6 +154,9 @@ case "$ARTIFACT" in
         make -C "deploy/packages/${PKGERDIR}" clean
         EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
         ;;
+    docker)
+        make_docker
+        ;;
     *)
         log "Unknown artifact $ARTIFACT"
         exit 1
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 9e0ba9d5b..78f1d9b87 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,10 +1,7 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64
-ARG RUN_FROM=alpine:3.12
+ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine
+ARG RUN_FROM=alpine:3.14
 FROM ${BUILD_FROM} AS builder
 
-ARG QEMU_ARCH=x86_64
-COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
-
 RUN apk add --no-cache \
     git \
     curl \
@@ -43,10 +40,9 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \
     org.label-schema.vcs-url="https://github.com/emqx/emqx" \
     maintainer="Raymond M Mouthaan <raymondmmouthaan@gmail.com>, Huang Rui <vowstar@gmail.com>, EMQ X Team <support@emqx.io>"
 
-ARG QEMU_ARCH=x86_64
 ARG EMQX_NAME=emqx
 
-COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
+COPY deploy/docker/docker-entrypoint.sh /usr/bin/
 COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
 
 RUN ln -s /opt/emqx/bin/* /usr/local/bin/
diff --git a/docker.mk b/docker.mk
deleted file mode 100644
index e2fe61d36..000000000
--- a/docker.mk
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-## default globals
-TARGET ?= emqx/emqx
-QEMU_ARCH ?= x86_64
-ARCH ?= amd64
-QEMU_VERSION ?= v5.0.0-2
-OS ?= alpine
-export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-
-ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
-	ARCH_LIST := amd64 arm64v8 arm32v7
-	EMQX_NAME := emqx-ee
-else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx-edge
-else
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx
-endif
-
-.PHONY: docker
-docker: docker-build docker-tag docker-save
-
-.PHONY: docker-prepare
-docker-prepare:
-	## Prepare the machine before any code installation scripts
-	# @echo "PREPARE: Setting up dependencies."
-	# @apt update -y
-	# @apt install --only-upgrade docker-ce -y
-
-	## Update docker configuration to enable docker manifest command
-	@echo "PREPARE: Updating docker configuration"
-	@mkdir -p $$HOME/.docker
-
-	# enable experimental to use docker manifest command
-	@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
-	# enable experimental
-	@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json 
-	@service docker restart
-
-.PHONY: docker-build
-docker-build:
-	## Build Docker image
-	@echo "DOCKER BUILD: Build Docker image."
-	@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
-	@echo "DOCKER BUILD: arch - $(ARCH)."
-	@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
-	@echo "DOCKER BUILD: docker repo - $(TARGET) "
-	@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
-
-	## Prepare qemu to build images other then x86_64 on travis
-	@echo "PREPARE: Qemu" \
-	&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-	@mkdir -p tmp \
-	&& cd tmp \
-	&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& cd -
-
-	@docker build --no-cache \
-		--build-arg PKG_VSN=$(PKG_VSN)  \
-		--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH)  \
-		--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
-		--build-arg EMQX_NAME=$(EMQX_NAME) \
-		--build-arg QEMU_ARCH=$(QEMU_ARCH) \
-		--tag $(TARGET):build-$(OS)-$(ARCH) \
-		-f deploy/docker/Dockerfile .
-
-.PHONY: docker-tag
-docker-tag:
-	@echo "DOCKER TAG: Tag Docker image."
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
-			docker tag $(TARGET):build-$(OS)-$${arch}  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-			echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
-			if [ $${arch} = amd64 ]; then \
-				docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
-				echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
-			fi; \
-		fi; \
-	done
-
-.PHONY: docker-save
-docker-save:
-	@echo "DOCKER SAVE: Save Docker image." 
-
-	@mkdir -p _packages/$(EMQX_NAME)
-
-	@if [ -n  "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
-		zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
-		mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
-	fi
-	
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker save  $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
-		fi; \
-	done
-
-.PHONY: docker-push
-docker-push:
-	@echo "DOCKER PUSH: Push Docker image."; 
-	@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN)."; 
-
-	@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker push $(TARGET):$(PKG_VSN); \
-		docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
-		docker push $(TARGET):latest; \
-	fi;
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker push  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-		fi; \
-	done
-
-.PHONY: docker-manifest-list
-docker-manifest-list:
-	version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
-	latest="docker manifest create --amend $(TARGET):latest"; \
-	for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
-			version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-			latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-		fi; \
-	done; \
-	eval $$version; \
-	eval $$latest; 
-
-	for arch in $(ARCH_LIST); do \
-		case $${arch} in \
-			"amd64") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-				fi; \
-				;; \
-			"arm64v8") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-				fi; \
-				;; \
-			"arm32v7") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-				fi; \
-				;; \
-			"i386") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-				fi; \
-				;; \
-			"s390x") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-				fi; \
-				;; \
-		esac; \
-	done; 
-
-	docker manifest inspect $(TARGET):$(PKG_VSN)
-	docker manifest push $(TARGET):$(PKG_VSN); 
-	docker manifest inspect $(TARGET):latest
-	docker manifest push $(TARGET):latest;
-
-.PHONY: docker-clean
-docker-clean:
-	@echo "DOCKER CLEAN: Clean Docker image."
-
-	@if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)); fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
-		fi \
-	done

From 83630982bcedc647fa5d35a72b66a8a2f4d78cde Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 3 Aug 2021 14:32:50 +0800
Subject: [PATCH 25/27] build(docker): delete etc path in volumes on Dockerfile

---
 deploy/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 78f1d9b87..623d01d96 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -58,7 +58,7 @@ RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \
 
 USER emqx
 
-VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"]
+VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
 
 # emqx will occupy these port:
 # - 1883 port for MQTT

From b16cf44bf63a9fcb98664d5100a426222124780d Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Mon, 2 Aug 2021 09:30:54 +0800
Subject: [PATCH 26/27] refactor(gw-lwm2m): refine lwm2m

---
 apps/emqx_gateway/etc/emqx_gateway.conf       |   30 +
 apps/emqx_gateway/src/emqx_gateway.app.src    |    2 +-
 apps/emqx_gateway/src/emqx_gateway_app.erl    |    9 +-
 apps/emqx_gateway/src/emqx_gateway_schema.erl |   24 +-
 .../src/lwm2m/emqx_lwm2m_coap_resource.erl    |    9 +-
 .../src/lwm2m/emqx_lwm2m_impl.erl             |  173 ++
 .../src/lwm2m/emqx_lwm2m_protocol.erl         |   38 +-
 .../src/lwm2m/emqx_lwm2m_xml_object_db.erl    |   12 +-
 .../src/lwm2m/test/emqx_lwm2m_SUITE.erl       | 1953 ----------------
 .../src/lwm2m/test/emqx_tlv_SUITE.erl         |  240 --
 .../src/lwm2m/test/test_mqtt_broker.erl       |  171 --
 apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl   | 1971 +++++++++++++++++
 apps/emqx_gateway/test/emqx_tlv_SUITE.erl     |  238 ++
 apps/emqx_gateway/test/test_mqtt_broker.erl   |  171 ++
 rebar.config.erl                              |    1 +
 15 files changed, 2651 insertions(+), 2391 deletions(-)
 create mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/test/emqx_lwm2m_SUITE.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl
 create mode 100644 apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
 create mode 100644 apps/emqx_gateway/test/emqx_tlv_SUITE.erl
 create mode 100644 apps/emqx_gateway/test/test_mqtt_broker.erl

diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf
index cdb776af1..b27e723a9 100644
--- a/apps/emqx_gateway/etc/emqx_gateway.conf
+++ b/apps/emqx_gateway/etc/emqx_gateway.conf
@@ -127,4 +127,34 @@ gateway: {
         #listener.udp.1: {}
         #listener.dtls.1: {}
     }
+
+    lwm2m_xml_dir: "{{ platform_etc_dir }}/lwm2m_xml"
+
+    lwm2m.1: {
+
+        lifetime_min: 1s
+
+        lifetime_max: 86400s
+
+        qmode_time_windonw: 22
+
+        auto_observe: false
+
+        mountpoint: "lwm2m/%e/"
+
+        ## always | contains_object_list
+        update_msg_publish_condition: contains_object_list
+
+        translators: {
+            command: "dn/#"
+            response: "up/resp"
+            notify: "up/notify"
+            register: "up/resp"
+            update: "up/resp"
+        }
+
+        listener.udp.1 {
+            bind: 5783
+        }
+    }
 }
diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src
index 541f31a23..e84ae0f7f 100644
--- a/apps/emqx_gateway/src/emqx_gateway.app.src
+++ b/apps/emqx_gateway/src/emqx_gateway.app.src
@@ -3,7 +3,7 @@
   {vsn, "0.1.0"},
   {registered, []},
   {mod, {emqx_gateway_app, []}},
-  {applications, [kernel, stdlib, grpc]},
+  {applications, [kernel, stdlib, grpc, lwm2m_coap]},
   {env, []},
   {modules, []},
   {licenses, ["Apache 2.0"]},
diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl
index 582a3d26a..f4918e75d 100644
--- a/apps/emqx_gateway/src/emqx_gateway_app.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_app.erl
@@ -44,7 +44,8 @@ load_default_gateway_applications() ->
 
 gateway_type_searching() ->
     %% FIXME: Hardcoded apps
-    [emqx_stomp_impl, emqx_sn_impl, emqx_exproto_impl, emqx_coap_impl].
+    [emqx_stomp_impl, emqx_sn_impl, emqx_exproto_impl,
+     emqx_coap_impl, emqx_lwm2m_impl].
 
 load(Mod) ->
     try
@@ -81,10 +82,14 @@ create_gateway_by_default([{Type, Name, Confs}|More]) ->
     create_gateway_by_default(More).
 
 zipped_confs() ->
-    All = maps:to_list(emqx_config:get([gateway])),
+    All = maps:to_list(
+            maps:without(exclude_options(), emqx_config:get([gateway]))),
     lists:append(lists:foldr(
       fun({Type, Gws}, Acc) ->
         {Names, Confs} = lists:unzip(maps:to_list(Gws)),
         Types = [ Type || _ <- lists:seq(1, length(Names))],
         [lists:zip3(Types, Names, Confs) | Acc]
       end, [], All)).
+
+exclude_options() ->
+   [lwm2m_xml_dir].
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index f5f2b9ab0..51b0f182b 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -34,8 +34,10 @@ structs() -> ["gateway"].
 fields("gateway") ->
     [{stomp, t(ref(stomp))},
      {mqttsn, t(ref(mqttsn))},
-     {exproto, t(ref(exproto))},
-     {coap, t(ref(coap))}
+     {coap, t(ref(coap))},
+     {lwm2m, t(ref(lwm2m))},
+     {lwm2m_xml_dir, t(string())},
+     {exproto, t(ref(exproto))}
     ];
 
 fields(stomp) ->
@@ -74,6 +76,21 @@ fields(mqttsn_predefined) ->
     , {topic, t(string())}
     ];
 
+fields(lwm2m) ->
+    [{"$id", t(ref(lwm2m_structs))}
+    ];
+
+fields(lwm2m_structs) ->
+    [ {lifetime_min, t(duration())}
+    , {lifetime_max, t(duration())}
+    , {qmode_time_windonw, t(integer())}
+    , {auto_observe, t(boolean())}
+    , {mountpoint, t(string())}
+    , {update_msg_publish_condition, t(union([always, contains_object_list]))}
+    , {translators, t(ref(translators))}
+    , {listener, t(ref(udp_listener_group))}
+    ];
+
 fields(exproto) ->
     [{"$id", t(ref(exproto_structs))}];
 
@@ -100,6 +117,9 @@ fields(clientinfo_override) ->
     , {clientid, t(string())}
     ];
 
+fields(translators) ->
+    [{"$name", t(string())}];
+
 fields(udp_listener_group) ->
     [ {udp, t(ref(udp_listener))}
     , {dtls, t(ref(dtls_listener))}
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl
index 8a7b41291..588dd523e 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl
@@ -363,9 +363,12 @@ check_epn(undefined) -> false;
 check_epn(_)         -> true.
 
 check_lifetime(undefined) -> false;
-check_lifetime(LifeTime) when is_integer(LifeTime) ->
-    Max = proplists:get_value(lifetime_max, lwm2m_coap_responder:options(), 315360000),
-    Min = proplists:get_value(lifetime_min, lwm2m_coap_responder:options(), 0),
+check_lifetime(LifeTime0) when is_integer(LifeTime0) ->
+    LifeTime = timer:seconds(LifeTime0),
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}),
+    Max = maps:get(lifetime_max, Envs, 315360000),
+    Min = maps:get(lifetime_min, Envs, 0),
+
     if
         LifeTime >= Min, LifeTime =< Max ->
             true;
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
new file mode 100644
index 000000000..d94c9fa8b
--- /dev/null
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
@@ -0,0 +1,173 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc The LwM2M Gateway Implement interface
+-module(emqx_lwm2m_impl).
+
+-behavior(emqx_gateway_impl).
+
+%% APIs
+-export([ load/0
+        , unload/0
+        ]).
+
+-export([]).
+
+-export([ init/1
+        , on_insta_create/3
+        , on_insta_update/4
+        , on_insta_destroy/3
+        ]).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+load() ->
+    RegistryOptions = [ {cbkmod, ?MODULE}
+                      ],
+    emqx_gateway_registry:load(lwm2m, RegistryOptions, []).
+
+unload() ->
+    %% XXX:
+    lwm2m_coap_server_registry:remove_handler(
+      [<<"rd">>],
+      emqx_lwm2m_coap_resource, undefined
+     ),
+    emqx_gateway_registry:unload(lwm2m).
+
+init(_) ->
+    %% Handler
+    _ = lwm2m_coap_server:start_registry(),
+    lwm2m_coap_server_registry:add_handler(
+      [<<"rd">>],
+      emqx_lwm2m_coap_resource, undefined
+     ),
+    %% Xml registry
+    {ok, _} = emqx_lwm2m_xml_object_db:start_link(
+                emqx_config:get([gateway, lwm2m_xml_dir])
+              ),
+
+    %% XXX: Self managed table?
+    %% TODO: Improve it later
+    {ok, _} = emqx_lwm2m_cm:start_link(),
+
+    GwState = #{},
+    {ok, GwState}.
+
+%% TODO: deinit
+
+%%--------------------------------------------------------------------
+%% emqx_gateway_registry callbacks
+%%--------------------------------------------------------------------
+
+on_insta_create(_Insta = #{ id := InstaId,
+                            rawconf := RawConf
+                          }, Ctx, _GwState) ->
+    Listeners = emqx_gateway_utils:normalize_rawconf(RawConf),
+    ListenerPids = lists:map(fun(Lis) ->
+                     start_listener(InstaId, Ctx, Lis)
+                   end, Listeners),
+    {ok, ListenerPids, _InstaState = #{ctx => Ctx}}.
+
+on_insta_update(NewInsta, OldInsta, GwInstaState = #{ctx := Ctx}, GwState) ->
+    InstaId = maps:get(id, NewInsta),
+    try
+        %% XXX: 1. How hot-upgrade the changes ???
+        %% XXX: 2. Check the New confs first before destroy old instance ???
+        on_insta_destroy(OldInsta, GwInstaState, GwState),
+        on_insta_create(NewInsta, Ctx, GwState)
+    catch
+        Class : Reason : Stk ->
+            logger:error("Failed to update stomp instance ~s; "
+                         "reason: {~0p, ~0p} stacktrace: ~0p",
+                         [InstaId, Class, Reason, Stk]),
+            {error, {Class, Reason}}
+    end.
+
+on_insta_destroy(_Insta = #{ id := InstaId,
+                             rawconf := RawConf
+                           }, _GwInstaState, _GwState) ->
+    Listeners = emqx_gateway_utils:normalize_rawconf(RawConf),
+    lists:foreach(fun(Lis) ->
+        stop_listener(InstaId, Lis)
+    end, Listeners).
+
+%%--------------------------------------------------------------------
+%% Internal funcs
+%%--------------------------------------------------------------------
+
+start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) ->
+    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
+    case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of
+        {ok, Pid} ->
+            io:format("Start lwm2m ~s:~s listener on ~s successfully.~n",
+                      [InstaId, Type, ListenOnStr]),
+            Pid;
+        {error, Reason} ->
+            io:format(standard_error,
+                      "Failed to start lwm2m ~s:~s listener on ~s: ~0p~n",
+                      [InstaId, Type, ListenOnStr, Reason]),
+            throw({badconf, Reason})
+    end.
+
+start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) ->
+    Name = name(InstaId, udp),
+    NCfg = Cfg#{ctx => Ctx},
+    NSocketOpts = merge_default(SocketOpts),
+    Options = [{config, NCfg}|NSocketOpts],
+    case Type of
+        udp ->
+            lwm2m_coap_server:start_udp(Name, ListenOn, Options);
+        dtls ->
+            lwm2m_coap_server:start_dtls(Name, ListenOn, Options)
+    end.
+
+name(InstaId, Type) ->
+    list_to_atom(lists:concat([InstaId, ":", Type])).
+
+merge_default(Options) ->
+    Default = emqx_gateway_utils:default_udp_options(),
+    case lists:keytake(udp_options, 1, Options) of
+        {value, {udp_options, TcpOpts}, Options1} ->
+            [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
+             | Options1];
+        false ->
+            [{udp_options, Default} | Options]
+    end.
+
+stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
+    StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
+    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
+    case StopRet of
+        ok -> io:format("Stop lwm2m ~s:~s listener on ~s successfully.~n",
+                        [InstaId, Type, ListenOnStr]);
+        {error, Reason} ->
+            io:format(standard_error,
+                      "Failed to stop lwm2m ~s:~s listener on ~s: ~0p~n",
+                      [InstaId, Type, ListenOnStr, Reason]
+                     )
+    end,
+    StopRet.
+
+stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) ->
+    Name = name(InstaId, Type),
+    case Type of
+        udp ->
+            lwm2m_coap_server:stop_udp(Name, ListenOn);
+        dtls ->
+            lwm2m_coap_server:stop_dtls(Name, ListenOn)
+    end.
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
index e13b19e0a..2ee61ec70 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
@@ -75,7 +75,8 @@ call(Pid, Msg, Timeout) ->
     end.
 
 init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) ->
-    Mountpoint = proplists:get_value(mountpoint, lwm2m_coap_responder:options(), ""),
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}),
+    Mountpoint = iolist_to_binary(maps:get(mountpoint, Envs, "")),
     Lwm2mState = #lwm2m_state{peername = Peername,
                               endpoint_name = EndpointName,
                               version = Ver,
@@ -89,7 +90,10 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">>
         ok ->
             _ = run_hooks('client.connack', [conninfo(Lwm2mState), success], undefined),
 
-            Sockport = proplists:get_value(port, lwm2m_coap_responder:options(), 5683),
+            %% FIXME:
+            Sockport = 5683,
+            %Sockport = proplists:get_value(port, lwm2m_coap_responder:options(), 5683),
+
             ClientInfo1 = maps:put(sockport, Sockport, ClientInfo),
             Lwm2mState1 = Lwm2mState#lwm2m_state{started_at = time_now(),
                                                  mountpoint = maps:get(mountpoint, ClientInfo1)},
@@ -124,8 +128,10 @@ update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, regi
                                                     coap_pid = CoapPid, endpoint_name = Epn}) ->
     UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo),
 
-    _ = case proplists:get_value(update_msg_publish_condition,
-            lwm2m_coap_responder:options(), contains_object_list) of
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}),
+
+    _ = case maps:get(update_msg_publish_condition,
+                      Envs, contains_object_list) of
         always ->
             send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState);
         contains_object_list ->
@@ -294,7 +300,8 @@ auto_observe_object_list(Expected, Registered) ->
 
 send_auto_observe(CoapPid, RegInfo, EndpointName) ->
     %% - auto observe the objects
-    case proplists:get_value(auto_observe, lwm2m_coap_responder:options(), false) of
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), 
+    case proplists:get_value(auto_observe, Envs, false) of
         false ->
             ?LOG(info, "Auto Observe Disabled", []);
         TrueOrObjList ->
@@ -379,7 +386,12 @@ get_cached_downlink_messages() ->
 is_cache_mode(RegInfo, StartedAt) ->
     case is_psm(RegInfo) orelse is_qmode(RegInfo) of
         true ->
-            QModeTimeWind = proplists:get_value(qmode_time_window, lwm2m_coap_responder:options(), 22),
+            Envs = proplists:get_value(
+                     config,
+                     lwm2m_coap_responder:options(),
+                     #{}
+                    ),
+            QModeTimeWind = maps:get(qmode_time_window, Envs, 22),
             Now = time_now(),
             if (Now - StartedAt) >= QModeTimeWind -> true;
                 true -> false
@@ -400,15 +412,17 @@ is_qmode(_) -> false.
 %%--------------------------------------------------------------------
 
 downlink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) ->
-    Topics = proplists:get_value(topics, lwm2m_coap_responder:options(), []),
-    DnTopic = proplists:get_value(downlink_topic_key(EventType), Topics,
-                                  default_downlink_topic(EventType)),
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}),
+    Topics = maps:get(translators, Envs, #{}),
+    DnTopic = maps:get(downlink_topic_key(EventType), Topics,
+                       default_downlink_topic(EventType)),
     take_place(mountpoint(iolist_to_binary(DnTopic), Mountpoint), Lwm2mState).
 
 uplink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) ->
-    Topics = proplists:get_value(topics, lwm2m_coap_responder:options(), []),
-    UpTopic = proplists:get_value(uplink_topic_key(EventType), Topics,
-                                  default_uplink_topic(EventType)),
+    Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}),
+    Topics = maps:get(translators, Envs, #{}),
+    UpTopic = maps:get(uplink_topic_key(EventType), Topics,
+                       default_uplink_topic(EventType)),
     take_place(mountpoint(iolist_to_binary(UpTopic), Mountpoint), Lwm2mState).
 
 downlink_topic_key(EventType) when is_binary(EventType) ->
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
index c7e68d281..ea5f878d3 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
@@ -22,7 +22,7 @@
 % This module is for future use. Disabled now.
 
 %% API
--export([ start_link/0
+-export([ start_link/1
         , stop/0
         , find_name/1
         , find_objectid/1
@@ -49,8 +49,8 @@
 %% API Function Definitions
 %% ------------------------------------------------------------------
 
-start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+start_link(XmlDir) ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []).
 
 find_objectid(ObjectId) ->
     ObjectIdInt =   case is_list(ObjectId) of
@@ -85,10 +85,10 @@ stop() ->
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------
 
-init([]) ->
+init([XmlDir]) ->
     _ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]),
     _ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]),
-    load(emqx_config:get([emqx_lwm2m, xml_dir])),
+    load(XmlDir),
     {ok, #state{}}.
 
 handle_call(_Request, _From, State) ->
@@ -108,8 +108,6 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-
-
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_gateway/src/lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/src/lwm2m/test/emqx_lwm2m_SUITE.erl
deleted file mode 100644
index 1e9c8f43c..000000000
--- a/apps/emqx_gateway/src/lwm2m/test/emqx_lwm2m_SUITE.erl
+++ /dev/null
@@ -1,1953 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_lwm2m_SUITE).
-
-% -compile(export_all).
-% -compile(nowarn_export_all).
-
-% -define(PORT, 5683).
-
-% -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)).
-
-% -include("src/lwm2m/include/emqx_lwm2m.hrl").
-% -include_lib("lwm2m_coap/include/coap.hrl").
-% -include_lib("eunit/include/eunit.hrl").
-% -include_lib("common_test/include/ct.hrl").
-
-% %%--------------------------------------------------------------------
-% %% Setups
-% %%--------------------------------------------------------------------
-
-% all() ->
-%     [ {group, test_grp_0_register}
-%     , {group, test_grp_1_read}
-%     , {group, test_grp_2_write}
-%     , {group, test_grp_3_execute}
-%     , {group, test_grp_4_discover}
-%     , {group, test_grp_5_write_attr}
-%     , {group, test_grp_6_observe}
-%     , {group, test_grp_8_object_19}
-%     ].
-
-% suite() -> [{timetrap, {seconds, 90}}].
-
-% groups() ->
-%     RepeatOpt = {repeat_until_all_ok, 1},
-%     [
-%         {test_grp_0_register, [RepeatOpt], [
-%             case01_register,
-%             case01_register_additional_opts,
-%             case01_register_incorrect_opts,
-%             case01_register_report,
-%             case02_update_deregister,
-%             case03_register_wrong_version,
-%             case04_register_and_lifetime_timeout,
-%             case05_register_wrong_epn,
-%             case06_register_wrong_lifetime,
-%             case07_register_alternate_path_01,
-%             case07_register_alternate_path_02,
-%             case08_reregister
-%         ]},
-%         {test_grp_1_read, [RepeatOpt], [
-%             case10_read,
-%             case10_read_separate_ack,
-%             case11_read_object_tlv,
-%             case11_read_object_json,
-%             case12_read_resource_opaque,
-%             case13_read_no_xml
-%         ]},
-%         {test_grp_2_write, [RepeatOpt], [
-%             case20_write,
-%             case21_write_object,
-%             case22_write_error,
-%             case20_single_write
-%         ]},
-%         {test_grp_create, [RepeatOpt], [
-%             case_create_basic
-%         ]},
-%         {test_grp_delete, [RepeatOpt], [
-%             case_delete_basic
-%         ]},
-%         {test_grp_3_execute, [RepeatOpt], [
-%             case30_execute, case31_execute_error
-%         ]},
-%         {test_grp_4_discover, [RepeatOpt], [
-%             case40_discover
-%         ]},
-%         {test_grp_5_write_attr, [RepeatOpt], [
-%             case50_write_attribute
-%         ]},
-%         {test_grp_6_observe, [RepeatOpt], [
-%             case60_observe
-%         ]},
-%         {test_grp_7_block_wize_transfer, [RepeatOpt], [
-%             case70_read_large, case70_write_large
-%         ]},
-%         {test_grp_8_object_19, [RepeatOpt], [
-%             case80_specail_object_19_1_0_write,
-%             case80_specail_object_19_0_0_notify
-%             %case80_specail_object_19_0_0_response,
-%             %case80_normal_object_19_0_0_read
-%         ]},
-%         {test_grp_9_psm_queue_mode, [RepeatOpt], [
-%             case90_psm_mode,
-%             case90_queue_mode
-%         ]}
-%     ].
-
-% init_per_suite(Config) ->
-%     emqx_ct_helpers:start_apps([emqx]),
-%     Config.
-
-% end_per_suite(Config) ->
-%     timer:sleep(300),
-%     emqx_ct_helpers:stop_apps([emqx]),
-%     Config.
-
-% init_per_testcase(_AllTestCase, Config) ->
-%     application:set_env(emqx_lwm2m, bind_udp, [{5683, []}]),
-%     application:set_env(emqx_lwm2m, bind_dtls, [{5684, []}]),
-%     application:set_env(emqx_lwm2m, xml_dir, emqx_ct_helpers:deps_path(emqx_lwm2m, "lwm2m_xml")),
-%     application:set_env(emqx_lwm2m, lifetime_max, 86400),
-%     application:set_env(emqx_lwm2m, lifetime_min, 1),
-%     application:set_env(emqx_lwm2m, mountpoint, "lwm2m/%e/"),
-%     {ok, _Started} = application:ensure_all_started(emqx_lwm2m),
-%     {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]),
-
-%     {ok, C} = emqtt:start_link([{host, "localhost"},{port, 1883},{clientid, <<"c1">>}]),
-%     {ok, _} = emqtt:connect(C),
-%     timer:sleep(100),
-
-%     [{sock, ClientUdpSock}, {emqx_c, C} | Config].
-
-% end_per_testcase(_AllTestCase, Config) ->
-%     timer:sleep(300),
-%     gen_udp:close(?config(sock, Config)),
-%     emqtt:disconnect(?config(emqx_c, Config)),
-%     ok = application:stop(emqx_lwm2m),
-%     ok = application:stop(lwm2m_coap).
-
-% %%--------------------------------------------------------------------
-% %% Cases
-% %%--------------------------------------------------------------------
-
-% case01_register(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-
-%     %% checkpoint 1 - response
-%     #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
-%         test_recv_coap_response(UdpSock),
-%     ack = Type,
-%     {ok, created} = Method,
-%     RspId = MsgId,
-%     Location = proplists:get_value(location_path, Opts),
-%     ?assertNotEqual(undefined, Location),
-
-%     %% checkpoint 2 - verify subscribed topics
-%     timer:sleep(50),
-%     ?LOGT("all topics: ~p", [test_mqtt_broker:get_subscrbied_topics()]),
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
-
-%     % ----------------------------------------
-%     % DE-REGISTER command
-%     % ----------------------------------------
-%     ?LOGT("start to send DE-REGISTER command", []),
-%     MsgId3 = 52,
-%     test_send_coap_request( UdpSock,
-%                             delete,
-%                             sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{payload = <<>>},
-%                             [],
-%                             MsgId3),
-%     #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
-%     {ok,deleted} = Method3,
-%     MsgId3 = RspId3,
-%     timer:sleep(50),
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case01_register_additional_opts(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-
-%     AddOpts = "ep=~s&lt=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2",
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-
-%     %% checkpoint 1 - response
-%     #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
-%         test_recv_coap_response(UdpSock),
-%     Type = ack,
-%     Method = {ok, created},
-%     RspId = MsgId,
-%     Location = proplists:get_value(location_path, Opts),
-%     ?assertNotEqual(undefined, Location),
-
-%     %% checkpoint 2 - verify subscribed topics
-%     timer:sleep(50),
-
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
-
-%     % ----------------------------------------
-%     % DE-REGISTER command
-%     % ----------------------------------------
-%     ?LOGT("start to send DE-REGISTER command", []),
-%     MsgId3 = 52,
-%     test_send_coap_request( UdpSock,
-%                             delete,
-%                             sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{payload = <<>>},
-%                             [],
-%                             MsgId3),
-%     #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
-%     {ok,deleted} = Method3,
-%     MsgId3 = RspId3,
-%     timer:sleep(50),
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case01_register_incorrect_opts(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-
-
-%     AddOpts = "ep=~s&lt=345&lwm2m=1&incorrect_opt",
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-
-%     %% checkpoint 1 - response
-%     #coap_message{type = ack, method = Method, id = MsgId} =
-%         test_recv_coap_response(UdpSock),
-%     ?assertEqual({error,bad_request}, Method).
-
-% case01_register_report(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-
-%     #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
-%         test_recv_coap_response(UdpSock),
-%     Type = ack,
-%     Method = {ok, created},
-%     RspId = MsgId,
-%     Location = proplists:get_value(location_path, Opts),
-%     ?assertNotEqual(undefined, Location),
-
-%     timer:sleep(50),
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"msgType">> => <<"register">>,
-%                                 <<"data">> => #{
-%                                     <<"alternatePath">> => <<"/">>,
-%                                     <<"ep">> => list_to_binary(Epn),
-%                                     <<"lt">> => 345,
-%                                     <<"lwm2m">> => <<"1">>,
-%                                     <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)),
-
-%     % ----------------------------------------
-%     % DE-REGISTER command
-%     % ----------------------------------------
-%     ?LOGT("start to send DE-REGISTER command", []),
-%     MsgId3 = 52,
-%     test_send_coap_request( UdpSock,
-%                             delete,
-%                             sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{payload = <<>>},
-%                             [],
-%                             MsgId3),
-%     #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
-%     {ok,deleted} = Method3,
-%     MsgId3 = RspId3,
-%     timer:sleep(50),
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case02_update_deregister(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-%     timer:sleep(100),
-%     #coap_message{type = ack, method = Method, options = Opts} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method),
-
-%     ?LOGT("Options got: ~p", [Opts]),
-%     Location = proplists:get_value(location_path, Opts),
-%     Register = emqx_json:encode(#{
-%                               <<"msgType">> => <<"register">>,
-%                               <<"data">> => #{
-%                                   <<"alternatePath">> => <<"/">>,
-%                                   <<"ep">> => list_to_binary(Epn),
-%                                   <<"lt">> => 345,
-%                                   <<"lwm2m">> => <<"1">>,
-%                                   <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>]
-%                               }
-%                             }),
-%     ?assertEqual(Register, test_recv_mqtt_response(ReportTopic)),
-
-%     % ----------------------------------------
-%     % UPDATE command
-%     % ----------------------------------------
-%     ?LOGT("start to send UPDATE command", []),
-%     MsgId2 = 27,
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>, </6>">>},
-%                             [],
-%                             MsgId2),
-%     #coap_message{type = ack, id = RspId2, method = Method2} = test_recv_coap_response(UdpSock),
-%     {ok,changed} = Method2,
-%     MsgId2 = RspId2,
-%     Update = emqx_json:encode(#{
-%                             <<"msgType">> => <<"update">>,
-%                             <<"data">> => #{
-%                                 <<"alternatePath">> => <<"/">>,
-%                                 <<"ep">> => list_to_binary(Epn),
-%                                 <<"lt">> => 789,
-%                                 <<"lwm2m">> => <<"1">>,
-%                                 <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>, <<"/6">>]
-%                             }
-%                         }),
-%     ?assertEqual(Update, test_recv_mqtt_response(ReportTopic)),
-
-%     % ----------------------------------------
-%     % DE-REGISTER command
-%     % ----------------------------------------
-%     ?LOGT("start to send DE-REGISTER command", []),
-%     MsgId3 = 52,
-%     test_send_coap_request( UdpSock,
-%                             delete,
-%                             sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{payload = <<>>},
-%                             [],
-%                             MsgId3),
-%     #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
-%     {ok,deleted} = Method3,
-%     MsgId3 = RspId3,
-
-%     timer:sleep(50),
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case03_register_wrong_version(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=8.3", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-%     #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({error,precondition_failed}, Method),
-%     timer:sleep(50),
-
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case04_register_and_lifetime_timeout(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=2&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-%     timer:sleep(100),
-%     #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method),
-
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
-
-%     % ----------------------------------------
-%     % lifetime timeout
-%     % ----------------------------------------
-%     timer:sleep(4000),
-
-%     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case05_register_wrong_epn(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     MsgId = 12,
-%     UdpSock = ?config(sock, Config),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?lt=345&lwm2m=1.0", [?PORT]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-%     #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({error,bad_request}, Method).
-
-% case06_register_wrong_lifetime(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId),
-%     #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({error,bad_request}, Method),
-%     timer:sleep(50),
-%     ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()).
-
-% case07_register_alternate_path_01(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>,
-%                             payload = <<"</>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
-%                             [],
-%                             MsgId),
-%     timer:sleep(50),
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case07_register_alternate_path_02(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>,
-%                             payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
-%                             [],
-%                             MsgId),
-%     timer:sleep(50),
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
-
-% case08_reregister(Config) ->
-%     % ----------------------------------------
-%     % REGISTER command
-%     % ----------------------------------------
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId = 12,
-%     SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
-%     ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>,
-%                             payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
-%                             [],
-%                             MsgId),
-%     timer:sleep(50),
-%     true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
-
-%     ReadResult = emqx_json:encode(
-%                             #{
-%                                 <<"msgType">> => <<"register">>,
-%                                 <<"data">> => #{
-%                                     <<"alternatePath">> => <<"/lwm2m">>,
-%                                     <<"ep">> => list_to_binary(Epn),
-%                                     <<"lt">> => 345,
-%                                     <<"lwm2m">> => <<"1">>,
-%                                     <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>]
-%                                  }
-%                              }
-%                             ),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)),
-%     timer:sleep(1000),
-
-%     %% the same lwm2mc client registers to server again
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>,
-%                             payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
-%                             [],
-%                             MsgId + 1),
-%     %% verify the lwm2m client is still online
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)).
-
-% case10_read(Config) ->
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-%     % step 1, device register ...
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>,
-%                             payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
-%                             [],
-%                             MsgId1),
-%     #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method1),
-%     test_recv_mqtt_response(RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 206,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     ?assertEqual(<<"/lwm2m/3/0/0">>, get_coap_path(Options2)),
-%     ?assertEqual(<<>>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"reqPath">> => <<"/3/0/0">>,
-%                                     <<"content">> => [#{
-%                                         path => <<"/3/0/0">>,
-%                                         value => <<"EMQ">>
-%                                     }]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case10_read_separate_ack(Config) ->
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     % step 1, device register ...
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 206,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command = #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     ?assertEqual(<<"/3/0/0">>, get_coap_path(Options2)),
-%     ?assertEqual(<<>>, Payload2),
-
-%     test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2),
-%     ReadResultACK = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"ack">>,
-%                                 <<"data">> => #{
-%                                     <<"path">> => <<"/3/0/0">>
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResultACK, test_recv_mqtt_response(RespTopic)),
-%     timer:sleep(100),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, false),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"reqPath">> => <<"/3/0/0">>,
-%                                     <<"content">> => [#{
-%                                         path => <<"/3/0/0">>,
-%                                         value => <<"EMQ">>
-%                                     }]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case11_read_object_tlv(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 207,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     timer:sleep(50),
-
-%     Tlv = <<16#08, 16#00, 16#3C, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>,
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+tlv">>, payload = Tlv}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"reqPath">> => <<"/3/0">>,
-%                                     <<"content">> => [
-%                                         #{
-%                                             path => <<"/3/0/0">>,
-%                                             value => <<"Open Mobile Alliance">>
-%                                         },
-%                                         #{
-%                                             path => <<"/3/0/1">>,
-%                                             value => <<"Lightweight M2M Client">>
-%                                         },
-%                                         #{
-%                                             path => <<"/3/0/2">>,
-%                                             value => <<"345000123">>
-%                                         }
-%                                     ]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case11_read_object_json(Config) ->
-%     % step 1, device register ...
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 206,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     timer:sleep(50),
-
-%     Json = <<"{\"bn\":\"/3/0\",\"e\":[{\"n\":\"0\",\"sv\":\"Open Mobile Alliance\"},{\"n\":\"1\",\"sv\":\"Lightweight M2M Client\"},{\"n\":\"2\",\"sv\":\"345000123\"}]}">>,
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+json">>, payload = Json}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"reqPath">> => <<"/3/0">>,
-%                                     <<"content">> => [
-%                                         #{
-%                                             path => <<"/3/0/0">>,
-%                                             value => <<"Open Mobile Alliance">>
-%                                         },
-%                                         #{
-%                                             path => <<"/3/0/1">>,
-%                                             value => <<"Lightweight M2M Client">>
-%                                         },
-%                                         #{
-%                                             path => <<"/3/0/2">>,
-%                                             value => <<"345000123">>
-%                                         }
-%                                     ]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case12_read_resource_opaque(Config) ->
-%     % step 1, device register ...
-%     UdpSock = ?config(sock, Config),
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 206,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0/8">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     timer:sleep(50),
-
-%     Opaque = <<20, 21, 22, 23>>,
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/octet-stream">>, payload = Opaque}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"reqPath">> => <<"/3/0/8">>,
-%                                     <<"content">> => [
-%                                         #{
-%                                             path => <<"/3/0/8">>,
-%                                             value => base64:encode(Opaque)
-%                                         }
-%                                     ]
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case13_read_no_xml(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a READ command to device
-%     CmdId = 206,
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/9723/0/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     ?LOGT("CommandJson=~p", [CommandJson]),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2} = Request2,
-%     ?LOGT("LwM2M client got ~p", [Request2]),
-
-%     ?assertEqual(get, Method2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"read">>,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/9723/0/0">>,
-%                                     <<"code">> => <<"4.00">>,
-%                                     <<"codeMsg">> => <<"bad_request">>
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case20_single_write(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/13">>,
-%                     <<"type">> => <<"Integer">>,
-%                     <<"value">> => <<"12345">>
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(put, Method2),
-%     ?assertEqual(<<"/3/0/13">>, Path2),
-%     Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16>>,
-%     ?assertEqual(Tlv_Value, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/13">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 },
-%                                 <<"msgType">> => <<"write">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case20_write(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write">>,
-%                 <<"data">> => #{
-%                     <<"basePath">> => <<"/3/0/13">>,
-%                     <<"content">> => [#{
-%                         type => <<"Float">>,
-%                         value => <<"12345.0">>
-%                     }]
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(put, Method2),
-%     ?assertEqual(<<"/3/0/13">>, Path2),
-%     Tlv_Value = <<200, 13, 8, 64,200,28,128,0,0,0,0>>,
-%     ?assertEqual(Tlv_Value, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     WriteResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/13">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 },
-%                                 <<"msgType">> => <<"write">>
-%                             }),
-%     ?assertEqual(WriteResult, test_recv_mqtt_response(RespTopic)).
-
-% case21_write_object(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write">>,
-%                 <<"data">> => #{
-%                     <<"basePath">> => <<"/3/0/">>,
-%                     <<"content">> => [#{
-%                         path => <<"13">>,
-%                         type => <<"Integer">>,
-%                         value => <<"12345">>
-%                     },#{
-%                         path => <<"14">>,
-%                         type => <<"String">>,
-%                         value => <<"87x">>
-%                     }]
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(post, Method2),
-%     ?assertEqual(<<"/3/0">>, Path2),
-%     Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16,
-%                   3:2, 0:1, 0:2, 3:3, 14, "87x">>,
-%     ?assertEqual(Tlv_Value, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"write">>,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 }
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case22_write_error(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write">>,
-%                 <<"data">> => #{
-%                     <<"basePath">> => <<"/3/0/1">>,
-%                     <<"content">> => [
-%                         #{
-%                             type => <<"Integer">>,
-%                             value => <<"12345">>
-%                         }
-%                     ]
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(put, Method2),
-%     ?assertEqual(<<"/3/0/1">>, Path2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, bad_request}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/1">>,
-%                                     <<"code">> => <<"4.00">>,
-%                                     <<"codeMsg">> => <<"bad_request">>
-%                                 },
-%                                 <<"msgType">> => <<"write">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case_create_basic(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2, send a CREATE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"create">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/5">>
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(post, Method2),
-%     ?assertEqual(<<"/5">>, Path2),
-%     ?assertEqual(<<"">>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, created}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/5">>,
-%                                     <<"code">> => <<"2.01">>,
-%                                     <<"codeMsg">> => <<"created">>
-%                                 },
-%                                 <<"msgType">> => <<"create">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case_delete_basic(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>, </5/0>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2, send a CREATE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"delete">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/5/0">>
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(delete, Method2),
-%     ?assertEqual(<<"/5/0">>, Path2),
-%     ?assertEqual(<<"">>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, deleted}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/5/0">>,
-%                                     <<"code">> => <<"2.02">>,
-%                                     <<"codeMsg">> => <<"deleted">>
-%                                 },
-%                                 <<"msgType">> => <<"delete">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case30_execute(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"execute">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/4">>,
-%                     %% "args" should not be present for "/3/0/4", only for testing the encoding here
-%                     <<"args">> => <<"2,7">>
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(post, Method2),
-%     ?assertEqual(<<"/3/0/4">>, Path2),
-%     ?assertEqual(<<"2,7">>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/4">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 },
-%                                 <<"msgType">> => <<"execute">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case31_execute_error(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"execute">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/4">>,
-%                     <<"args">> => <<"2,7">>
-%                 }
-%                },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(post, Method2),
-%     ?assertEqual(<<"/3/0/4">>, Path2),
-%     ?assertEqual(<<"2,7">>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, uauthorized}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/4">>,
-%                                     <<"code">> => <<"4.01">>,
-%                                     <<"codeMsg">> => <<"uauthorized">>
-%                                 },
-%                                 <<"msgType">> => <<"execute">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case40_discover(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"discover">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/7">>
-%                 } },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(get, Method2),
-%     ?assertEqual(<<"/3/0/7">>, Path2),
-%     ?assertEqual(<<>>, Payload2),
-%     timer:sleep(50),
-
-%     PayloadDiscover = <<"</3/0/7>;dim=8;pmin=10;pmax=60;gt=50;lt=42.2,</3/0/8>">>,
-%     test_send_coap_response(UdpSock,
-%                             "127.0.0.1",
-%                             ?PORT,
-%                             {ok, content},
-%                             #coap_content{content_format = <<"application/link-format">>, payload = PayloadDiscover},
-%                             Request2,
-%                             true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"discover">>,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/7">>,
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"content">> =>
-%                                         [<<"</3/0/7>;dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"</3/0/8>">>]
-%                                 }
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case50_write_attribute(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write-attr">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/9">>,
-%                     <<"pmin">> => <<"1">>,
-%                     <<"pmax">> => <<"5">>,
-%                     <<"lt">> => <<"5">>
-%                 } },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(100),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     ?LOGT("got options: ~p", [Options2]),
-%     Path2 = get_coap_path(Options2),
-%     Query2 = lists:sort(get_coap_query(Options2)),
-%     ?assertEqual(put, Method2),
-%     ?assertEqual(<<"/3/0/9">>, Path2),
-%     ?assertEqual(lists:sort([<<"pmax=5">>,<<"lt=5">>,<<"pmin=1">>]), Query2),
-%     ?assertEqual(<<>>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock,
-%                             "127.0.0.1",
-%                             ?PORT,
-%                             {ok, changed},
-%                             #coap_content{},
-%                             Request2,
-%                             true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/9">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 },
-%                                 <<"msgType">> => <<"write-attr">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case60_observe(Config) ->
-%     % step 1, device register ...
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     RespTopicAD = list_to_binary("lwm2m/"++Epn++"/up/notify"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopicAD, qos0),
-%     timer:sleep(200),
-
-%     std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
-
-%     % step2,  send a OBSERVE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"observe">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/10">>
-%                 }
-%              },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     Observe = get_coap_observe(Options2),
-%     ?assertEqual(get, Method2),
-%     ?assertEqual(<<"/3/0/10">>, Path2),
-%     ?assertEqual(Observe, 0),
-%     ?assertEqual(<<>>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_observe_ack( UdpSock,
-%                                 "127.0.0.1",
-%                                 ?PORT,
-%                                 {ok, content},
-%                                 #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>},
-%                                 Request2),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"observe">>,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/10">>,
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"content">> => [#{
-%                                         path => <<"/3/0/10">>,
-%                                         value => 2048
-%                                     }]
-%                                 }
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)),
-
-%     %% step3 the notifications
-%     timer:sleep(200),
-%     ObSeq = 3,
-%     test_send_coap_notif(   UdpSock,
-%                             "127.0.0.1",
-%                             ?PORT,
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"4096">>},
-%                             ObSeq,
-%                             Request2),
-%     timer:sleep(100),
-%     #coap_message{} = test_recv_coap_response(UdpSock),
-
-%     ReadResult2 = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"msgType">> => <<"notify">>,
-%                                 <<"seqNum">> => ObSeq,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/10">>,
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"content">> => [#{
-%                                         path => <<"/3/0/10">>,
-%                                         value => 4096
-%                                     }]
-%                                 }
-%                             }),
-%     ?assertEqual(ReadResult2, test_recv_mqtt_response(RespTopicAD)),
-
-%     %% Step3. cancel observe
-%     CmdId3 = 308,
-%     Command3 = #{<<"requestID">> => CmdId3, <<"cacheID">> => CmdId3,
-%                 <<"msgType">> => <<"cancel-observe">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/3/0/10">>
-%                 }
-%              },
-%     CommandJson3 = emqx_json:encode(Command3),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson3, 0),
-%     timer:sleep(50),
-%     Request3 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method3, options=Options3, payload=Payload3} = Request3,
-%     Path3 = get_coap_path(Options3),
-%     Observe3 = get_coap_observe(Options3),
-%     ?assertEqual(get, Method3),
-%     ?assertEqual(<<"/3/0/10">>, Path3),
-%     ?assertEqual(Observe3, 1),
-%     ?assertEqual(<<>>, Payload3),
-%     timer:sleep(50),
-
-%     test_send_coap_observe_ack( UdpSock,
-%                                 "127.0.0.1",
-%                                 ?PORT,
-%                                 {ok, content},
-%                                 #coap_content{content_format = <<"text/plain">>, payload = <<"1150">>},
-%                                 Request3),
-%     timer:sleep(100),
-
-%     ReadResult3 = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3,
-%                                 <<"msgType">> => <<"cancel-observe">>,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/3/0/10">>,
-%                                     <<"code">> => <<"2.05">>,
-%                                     <<"codeMsg">> => <<"content">>,
-%                                     <<"content">> => [#{
-%                                         path => <<"/3/0/10">>,
-%                                         value => 1150
-%                                     }]
-%                                 }
-%                             }),
-%     ?assertEqual(ReadResult3, test_recv_mqtt_response(RespTopic)).
-
-% case80_specail_object_19_0_0_notify(Config) ->
-%     % step 1, device register, with extra register options
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId1),
-%     #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method1),
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"msgType">> => <<"register">>,
-%                                 <<"data">> => #{
-%                                     <<"alternatePath">> => <<"/">>,
-%                                     <<"ep">> => list_to_binary(Epn),
-%                                     <<"lt">> => 345,
-%                                     <<"lwm2m">> => <<"1">>,
-%                                     <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>],
-%                                     <<"apn">> => <<"psmA.eDRX0.ctnb">>,
-%                                     <<"im">> => <<"13456">>,
-%                                     <<"ct">> => <<"2.0">>,
-%                                     <<"mt">> => <<"MDM9206">>,
-%                                     <<"mv">> => <<"4.0">>
-%                                 }
-%                              }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)),
-
-%     % step2,  send a OBSERVE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"observe">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/19/0/0">>
-%                 }
-%              },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     Observe = get_coap_observe(Options2),
-%     ?assertEqual(get, Method2),
-%     ?assertEqual(<<"/19/0/0">>, Path2),
-%     ?assertEqual(Observe, 0),
-%     ?assertEqual(<<>>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_observe_ack( UdpSock,
-%                                 "127.0.0.1",
-%                                 ?PORT,
-%                                 {ok, content},
-%                                 #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>},
-%                                 Request2),
-%     timer:sleep(100).
-
-%     %% step 3, device send uplink data notifications
-
-% case80_specail_object_19_1_0_write(Config) ->
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0",
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId1),
-%     #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method1),
-%     test_recv_mqtt_response(RespTopic),
-
-%     % step2,  send a WRITE command to device
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     CmdId = 307,
-%     Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                 <<"msgType">> => <<"write">>,
-%                 <<"data">> => #{
-%                     <<"path">> => <<"/19/1/0">>,
-%                     <<"type">> => <<"Opaque">>,
-%                     <<"value">> => base64:encode(<<12345:32>>)
-%                 }
-%                },
-
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50),
-%     Request2 = test_recv_coap_request(UdpSock),
-%     #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
-%     Path2 = get_coap_path(Options2),
-%     ?assertEqual(put, Method2),
-%     ?assertEqual(<<"/19/1/0">>, Path2),
-%     ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2),
-%     timer:sleep(50),
-
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
-%     timer:sleep(100),
-
-%     ReadResult = emqx_json:encode(#{
-%                                 <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                                 <<"data">> => #{
-%                                     <<"reqPath">> => <<"/19/1/0">>,
-%                                     <<"code">> => <<"2.04">>,
-%                                     <<"codeMsg">> => <<"changed">>
-%                                 },
-%                                 <<"msgType">> => <<"write">>
-%                             }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% case90_psm_mode(Config) ->
-%     server_cache_mode(Config, "ep=~s&lt=345&lwm2m=1&apn=psmA.eDRX0.ctnb").
-
-% case90_queue_mode(Config) ->
-%     server_cache_mode(Config, "ep=~s&lt=345&lwm2m=1&b=UQ").
-
-% server_cache_mode(Config, RegOption) ->
-%     application:set_env(?APP, qmode_time_window, 2),
-
-%     % step 1, device register, with apn indicates "PSM" mode
-%     Epn = "urn:oma:lwm2m:oma:3",
-
-%     MsgId1 = 15,
-%     UdpSock = ?config(sock, Config),
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
-%     timer:sleep(200),
-
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?"++RegOption, [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
-%                             [],
-%                             MsgId1),
-%     #coap_message{type = ack, method = Method1, options = Opts} = test_recv_coap_response(UdpSock),
-%     ?assertEqual({ok,created}, Method1),
-%     ?LOGT("Options got: ~p", [Opts]),
-%     Location = proplists:get_value(location_path, Opts),
-%     test_recv_mqtt_response(RespTopic),
-
-%     %% server not in PSM mode
-%     send_read_command_1(0, UdpSock),
-%     verify_read_response_1(0, UdpSock),
-
-%     %% server inters into PSM mode
-%     timer:sleep(2),
-
-%     %% verify server caches downlink commands
-%     send_read_command_1(1, UdpSock),
-%     send_read_command_1(2, UdpSock),
-%     send_read_command_1(3, UdpSock),
-
-%     ?assertEqual(timeout_test_recv_coap_request, test_recv_coap_request(UdpSock)),
-
-%     device_update_1(UdpSock, Location),
-
-%     verify_read_response_1(1, UdpSock),
-%     verify_read_response_1(2, UdpSock),
-%     verify_read_response_1(3, UdpSock).
-
-% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-% %%% Internal Functions
-% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-% send_read_command_1(CmdId, _UdpSock) ->
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
-%     Command =   #{
-%                     <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                     <<"msgType">> => <<"read">>,
-%                     <<"data">> => #{
-%                         <<"path">> => <<"/3/0/0">>
-%                     }
-%                 },
-%     CommandJson = emqx_json:encode(Command),
-%     test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
-%     timer:sleep(50).
-
-% verify_read_response_1(CmdId, UdpSock) ->
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-
-%     %% device receives a command
-%     Request = test_recv_coap_request(UdpSock),
-%     ?LOGT("LwM2M client got ~p", [Request]),
-
-%     %% device replies the commond
-%     test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request, true),
-
-%     ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
-%                             <<"msgType">> => <<"read">>,
-%                             <<"data">> => #{
-%                                 <<"code">> => <<"2.05">>,
-%                                 <<"codeMsg">> => <<"content">>,
-%                                 <<"content">> => [#{
-%                                     path => <<"/3/0/0">>,
-%                                     value => <<"EMQ">>
-%                                 }]
-%                             }
-%                          }),
-%     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
-
-% device_update_1(UdpSock, Location) ->
-%     Epn = "urn:oma:lwm2m:oma:3",
-%     RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
-%     ?LOGT("send UPDATE command", []),
-%     MsgId2 = 27,
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]),
-%                             #coap_content{payload = <<>>},
-%                             [],
-%                             MsgId2),
-%     #coap_message{type = ack, id = MsgId2, method = Method2} = test_recv_coap_response(UdpSock),
-%     {ok,changed} = Method2,
-%     test_recv_mqtt_response(RespTopic).
-
-% test_recv_mqtt_response(RespTopic) ->
-%     receive
-%         {publish, #{topic := RespTopic, payload := RM}} ->
-%             ?LOGT("test_recv_mqtt_response Response=~p", [RM]),
-%             RM
-%     after 1000 -> timeout_test_recv_mqtt_response
-%     end.
-
-% test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) ->
-%     is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
-%     is_list(Options) orelse error("Options must be a list"),
-%     case resolve_uri(Uri) of
-%         {coap, {IpAddr, Port}, Path, Query} ->
-%             Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]),
-%             Request = Request0#coap_message{id = MsgId},
-%             ?LOGT("send_coap_request Request=~p", [Request]),
-%             RequestBinary = lwm2m_coap_message_parser:encode(Request),
-%             ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]),
-%             ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary);
-%         {SchemeDiff, ChIdDiff, _, _} ->
-%             error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff])))
-%     end.
-
-% test_recv_coap_response(UdpSock) ->
-%     {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000),
-%     Response = lwm2m_coap_message_parser:decode(Packet),
-%     ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]),
-%     #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response,
-%     ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]),
-%     Response.
-
-% test_recv_coap_request(UdpSock) ->
-%     case gen_udp:recv(UdpSock, 0, 2000) of
-%         {ok, {_Address, _Port, Packet}} ->
-%             Request = lwm2m_coap_message_parser:decode(Packet),
-%             #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request,
-%             ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]),
-%             Request;
-%         {error, Reason} ->
-%             ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]),
-%             timeout_test_recv_coap_request
-%     end.
-
-% test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) ->
-%     is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
-%     is_list(Host) orelse error("Host is not a string"),
-
-%     {ok, IpAddr} = inet:getaddr(Host, inet),
-%     Response = lwm2m_coap_message:response(Code, Content, Request),
-%     Response2 = case Ack of
-%                     true -> Response#coap_message{type = ack};
-%                     false -> Response
-%                 end,
-%     ?LOGT("test_send_coap_response Response=~p", [Response2]),
-%     ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)).
-
-% test_send_empty_ack(UdpSock, Host, Port, Request) ->
-%     is_list(Host) orelse error("Host is not a string"),
-%     {ok, IpAddr} = inet:getaddr(Host, inet),
-%     EmptyACK = lwm2m_coap_message:ack(Request),
-%     ?LOGT("test_send_empty_ack EmptyACK=~p", [EmptyACK]),
-%     ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(EmptyACK)).
-
-% test_send_coap_observe_ack(UdpSock, Host, Port, Code, Content, Request) ->
-%     is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
-%     is_list(Host) orelse error("Host is not a string"),
-
-%     {ok, IpAddr} = inet:getaddr(Host, inet),
-%     Response = lwm2m_coap_message:response(Code, Content, Request),
-%     Response1 = lwm2m_coap_message:set(observe, 0, Response),
-%     Response2 = Response1#coap_message{type = ack},
-
-%     ?LOGT("test_send_coap_observe_ack Response=~p", [Response2]),
-%     ResponseBinary = lwm2m_coap_message_parser:encode(Response2),
-%     ok = gen_udp:send(UdpSock, IpAddr, Port, ResponseBinary).
-
-% test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) ->
-%     is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
-%     is_list(Host) orelse error("Host is not a string"),
-
-%     {ok, IpAddr} = inet:getaddr(Host, inet),
-%     Notif = lwm2m_coap_message:response({ok, content}, Content, Request),
-%     NewNotif = lwm2m_coap_message:set(observe, ObSeq, Notif),
-%     ?LOGT("test_send_coap_notif Response=~p", [NewNotif]),
-%     NotifBinary = lwm2m_coap_message_parser:encode(NewNotif),
-%     ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, NotifBinary]),
-%     ok = gen_udp:send(UdpSock, IpAddr, Port, NotifBinary).
-
-% std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) ->
-%     test_send_coap_request( UdpSock,
-%                             post,
-%                             sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
-%                             #coap_content{content_format = <<"text/plain">>, payload = ObjectList},
-%                             [],
-%                             MsgId1),
-%     #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock),
-%     test_recv_mqtt_response(RespTopic),
-%     timer:sleep(100).
-
-% resolve_uri(Uri) ->
-%     {ok, #{scheme := Scheme,
-%            host := Host,
-%            port := PortNo,
-%            path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri),
-%     Query = maps:get(query, URIMap, ""),
-%     {ok, PeerIP} = inet:getaddr(Host, inet),
-%     {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}.
-
-% split_path([]) -> [];
-% split_path([$/]) -> [];
-% split_path([$/ | Path]) -> split_segments(Path, $/, []).
-
-% split_query([]) -> [];
-% split_query(Path) -> split_segments(Path, $&, []).
-
-% split_segments(Path, Char, Acc) ->
-%     case string:rchr(Path, Char) of
-%         0 ->
-%             [make_segment(Path) | Acc];
-%         N when N > 0 ->
-%             split_segments(string:substr(Path, 1, N-1), Char,
-%                 [make_segment(string:substr(Path, N+1)) | Acc])
-%     end.
-
-% make_segment(Seg) ->
-%     list_to_binary(emqx_http_lib:uri_decode(Seg)).
-
-
-% get_coap_path(Options) ->
-%     get_path(Options, <<>>).
-
-% get_coap_query(Options) ->
-%     proplists:get_value(uri_query, Options, []).
-
-% get_coap_observe(Options) ->
-%     get_observe(Options).
-
-
-% get_path([], Acc) ->
-%     %?LOGT("get_path Acc=~p", [Acc]),
-%     Acc;
-% get_path([{uri_path, Path1}|T], Acc) ->
-%     %?LOGT("Path=~p, Acc=~p", [Path1, Acc]),
-%     get_path(T, join_path(Path1, Acc));
-% get_path([{_, _}|T], Acc) ->
-%     get_path(T, Acc).
-
-% get_observe([]) ->
-%     undefined;
-% get_observe([{observe, V}|_T]) ->
-%     V;
-% get_observe([{_, _}|T]) ->
-%     get_observe(T).
-
-% join_path([], Acc) -> Acc;
-% join_path([<<"/">>|T], Acc) ->
-%     join_path(T, Acc);
-% join_path([H|T], Acc) ->
-%     join_path(T, <<Acc/binary, $/, H/binary>>).
-
-% sprintf(Format, Args) ->
-%     lists:flatten(io_lib:format(Format, Args)).
diff --git a/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl
deleted file mode 100644
index d4d3c70cf..000000000
--- a/apps/emqx_gateway/src/lwm2m/test/emqx_tlv_SUITE.erl
+++ /dev/null
@@ -1,240 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_tlv_SUITE).
-
-% -compile(export_all).
-% -compile(nowarn_export_all).
-
-% -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)).
-
-% -include("src/lwm2m/include/emqx_lwm2m.hrl").
-% -include_lib("lwm2m_coap/include/coap.hrl").
-% -include_lib("eunit/include/eunit.hrl").
-
-
-% all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09].
-
-
-
-% init_per_suite(Config) ->
-%     Config.
-
-% end_per_suite(Config) ->
-%     Config.
-
-
-% case01(_Config) ->
-%     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case02(_Config) ->
-%     Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_multiple_resource => 16#06, value => [
-%                                                                     #{tlv_resource_instance => 16#00, value => <<1>>},
-%                                                                     #{tlv_resource_instance => 16#01, value => <<5>>}
-%             ]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case03(_Config) ->
-%     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
-%             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
-%             #{tlv_resource_with_value => 16#02, value => <<"345000123">>}
-%             ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case03_0(_Config) ->
-%     Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_multiple_resource => 16#02, value => [
-%             #{tlv_resource_instance => 16#7F, value => <<16#07>>},
-%             #{tlv_resource_instance => 16#0136, value => <<16#01>>}
-%         ]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case04(_Config) ->
-%     % 6.4.3.1 Single Object Instance Request Example
-%     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
-%             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
-%             #{tlv_resource_with_value => 16#02, value => <<"345000123">>},
-%             #{tlv_resource_with_value => 16#03, value => <<"1.0">>},
-%             #{tlv_multiple_resource => 16#06, value => [
-%                                                                         #{tlv_resource_instance => 16#00, value => <<1>>},
-%                                                                         #{tlv_resource_instance => 16#01, value => <<5>>}
-%                 ]},
-%             #{tlv_multiple_resource => 16#07, value => [
-%                                                                         #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>},
-%                                                                         #{tlv_resource_instance => 16#01, value => <<16#1388:16>>}
-%                 ]},
-%             #{tlv_multiple_resource => 16#08, value => [
-%                                                                         #{tlv_resource_instance => 16#00, value => <<16#7d>>},
-%                                                                         #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#09, value => <<16#64>>},
-%             #{tlv_resource_with_value => 16#0A, value => <<16#0F>>},
-%             #{tlv_multiple_resource => 16#0B, value => [
-%                 #{tlv_resource_instance => 16#00, value => <<16#00>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>},
-%             #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>},
-%             #{tlv_resource_with_value => 16#10, value => <<"U">>}
-%           ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case05(_Config) ->
-%     % 6.4.3.2 Multiple Object Instance Request Examples
-%     %   A) Request on Single-Instance Object
-%     Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_object_instance => 16#00, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
-%             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
-%             #{tlv_resource_with_value => 16#02, value => <<"345000123">>},
-%             #{tlv_resource_with_value => 16#03, value => <<"1.0">>},
-%             #{tlv_multiple_resource => 16#06, value => [
-%                 #{tlv_resource_instance => 16#00, value => <<1>>},
-%                 #{tlv_resource_instance => 16#01, value => <<5>>}
-%             ]},
-%             #{tlv_multiple_resource => 16#07, value => [
-%                 #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>},
-%                 #{tlv_resource_instance => 16#01, value => <<16#1388:16>>}
-%             ]},
-%             #{tlv_multiple_resource => 16#08, value => [
-%                 #{tlv_resource_instance => 16#00, value => <<16#7d>>},
-%                 #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#09, value => <<16#64>>},
-%             #{tlv_resource_with_value => 16#0A, value => <<16#0F>>},
-%             #{tlv_multiple_resource => 16#0B, value => [
-%                 #{tlv_resource_instance => 16#00, value => <<16#00>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>},
-%             #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>},
-%             #{tlv_resource_with_value => 16#10, value => <<"U">>}
-%         ]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case06(_Config) ->
-%     % 6.4.3.2 Multiple Object Instance Request Examples
-%     %   B) Request on Multiple-Instances Object having 2 instances
-%     Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_object_instance => 16#00, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<16#01>>},
-%             #{tlv_resource_with_value => 16#01, value => <<16#00>>},
-%             #{tlv_multiple_resource => 16#02, value => [
-%                 #{tlv_resource_instance => 16#7F, value => <<16#07>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#03, value => <<16#7F>>}
-%         ]},
-%         #{tlv_object_instance => 16#02, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<16#03>>},
-%             #{tlv_resource_with_value => 16#01, value => <<16#00>>},
-%             #{tlv_multiple_resource => 16#02, value => [
-%                 #{tlv_resource_instance => 16#7F, value => <<16#07>>},
-%                 #{tlv_resource_instance => 16#0136, value => <<16#01>>}
-%             ]},
-%             #{tlv_resource_with_value => 16#03, value => <<16#7F>>}
-%         ]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case07(_Config) ->
-%     % 6.4.3.2 Multiple Object Instance Request Examples
-%     %   C) Request on Multiple-Instances Object having 1 instance only
-%     Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_object_instance => 16#00, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<16#01>>},
-%             #{tlv_resource_with_value => 16#01, value => <<86400:32>>},
-%             #{tlv_resource_with_value => 16#06, value => <<16#01>>},
-%             #{tlv_resource_with_value => 16#07, value => <<$U>>}]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case08(_Config) ->
-%     % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource
-%     %   Example 1) request to Object 65 Instance 0: Read /65/0
-%     Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_multiple_resource => 16#00, value => [
-%             #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>},
-%             #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>}
-%             ]},
-%         #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>},
-%         #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
-% case09(_Config) ->
-%     % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource
-%     %   Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances
-%     Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>,
-%     R = emqx_lwm2m_tlv:parse(Data),
-%     Exp = [
-%         #{tlv_object_instance => 16#00, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<"myService 1">>},
-%             #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>},
-%             #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>}
-%         ]},
-%         #{tlv_object_instance => 16#01, value => [
-%             #{tlv_resource_with_value => 16#00, value => <<"myService 2">>},
-%             #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>},
-%             #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>}
-%         ]}
-%     ],
-%     ?assertEqual(Exp, R),
-%     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
-%     ?assertEqual(EncodedBinary, Data).
-
diff --git a/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl b/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl
deleted file mode 100644
index 4a6c77444..000000000
--- a/apps/emqx_gateway/src/lwm2m/test/test_mqtt_broker.erl
+++ /dev/null
@@ -1,171 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(test_mqtt_broker).
-
-% -compile(nowarn_export_all).
-% -compile(export_all).
-
-% -define(LOGT(Format, Args), logger:debug("TEST_BROKER: " ++ Format, Args)).
-
-% -record(state, {subscriber}).
-
-% -include_lib("emqx/include/emqx.hrl").
-
-% -include_lib("emqx/include/emqx_mqtt.hrl").
-
-% -include_lib("eunit/include/eunit.hrl").
-
-% start(_, <<"attacker">>, _, _, _) ->
-%     {stop, auth_failure};
-% start(ClientId, Username, Password, _Channel, KeepaliveInterval) ->
-%     true = is_binary(ClientId),
-%     (true = ( is_binary(Username)) orelse (Username == undefined) ),
-%     (true = ( is_binary(Password)) orelse (Password == undefined) ),
-%     self() ! {keepalive, start, KeepaliveInterval},
-%     {ok, []}.
-
-% publish(Topic, Payload, Qos) ->
-%     ClientId = <<"lwm2m_test_suite">>,
-%     Msg = emqx_message:make(ClientId, Qos, Topic, Payload),
-%     emqx:publish(Msg).
-
-% subscribe(Topic) ->
-%     gen_server:call(?MODULE, {subscribe, Topic, self()}).
-
-% unsubscribe(Topic) ->
-%     gen_server:call(?MODULE, {unsubscribe, Topic}).
-
-% get_subscrbied_topics() ->
-%     [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)].
-
-% start_link() ->
-%     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-% stop() ->
-%     gen_server:stop(?MODULE).
-
-% init(_Param) ->
-%     {ok, #state{subscriber = []}}.
-
-% handle_call({subscribe, Topic, Proc}, _From, State=#state{subscriber = SubList}) ->
-%     ?LOGT("test broker subscribe Topic=~p, Pid=~p~n", [Topic, Proc]),
-%     is_binary(Topic) orelse error("Topic should be a binary"),
-%     {reply, {ok, []}, State#state{subscriber = [{Topic, Proc}|SubList]}};
-
-% handle_call(get_subscribed_topics, _From, State=#state{subscriber = SubList}) ->
-%     Response = subscribed_topics(SubList, []),
-%     ?LOGT("test broker get subscribed topics=~p~n", [Response]),
-%     {reply, Response, State};
-
-% handle_call({unsubscribe, Topic}, _From, State=#state{subscriber = SubList}) ->
-%     ?LOGT("test broker unsubscribe Topic=~p~n", [Topic]),
-%     is_binary(Topic) orelse error("Topic should be a binary"),
-%     NewSubList = proplists:delete(Topic, SubList),
-%     {reply, {ok, []}, State#state{subscriber = NewSubList}};
-
-
-% handle_call({publish, {Topic, Msg, MatchedTopicFilter}}, _From, State=#state{subscriber = SubList}) ->
-%     (is_binary(Topic) and is_binary(Msg)) orelse error("Topic and Msg should be binary"),
-%     Pid = proplists:get_value(MatchedTopicFilter, SubList),
-%     ?LOGT("test broker publish topic=~p, Msg=~p, Pid=~p, MatchedTopicFilter=~p, SubList=~p~n", [Topic, Msg, Pid, MatchedTopicFilter, SubList]),
-%     (Pid == undefined) andalso ?LOGT("!!!!! this topic ~p has never been subscribed, please specify a valid topic filter", [MatchedTopicFilter]),
-%     ?assertNotEqual(undefined, Pid),
-%     Pid ! {deliver, #message{topic = Topic, payload = Msg}},
-%     {reply, ok, State};
-
-% handle_call(stop, _From, State) ->
-%     {stop, normal, stopped, State};
-
-% handle_call(Req, _From, State) ->
-%     ?LOGT("test_broker_server: ignore call Req=~p~n", [Req]),
-%     {reply, {error, badreq}, State}.
-
-
-% handle_cast(Msg, State) ->
-%     ?LOGT("test_broker_server: ignore cast msg=~p~n", [Msg]),
-%     {noreply, State}.
-
-% handle_info(Info, State) ->
-%     ?LOGT("test_broker_server: ignore info=~p~n", [Info]),
-%     {noreply, State}.
-
-% terminate(Reason, _State) ->
-%     ?LOGT("test_broker_server: terminate Reason=~p~n", [Reason]),
-%     ok.
-
-% code_change(_OldVsn, State, _Extra) ->
-%     {ok, State}.
-
-
-
-
-% subscribed_topics([], Acc) ->
-%     Acc;
-% subscribed_topics([{Topic,_Pid}|T], Acc) ->
-%     subscribed_topics(T, [Topic|Acc]).
-
-
-
-
-% -record(keepalive, {statfun, statval, tsec, tmsg, tref, repeat = 0}).
-
-% -type(keepalive() :: #keepalive{}).
-
-% %% @doc Start a keepalive
-% -spec(start(fun(), integer(), any()) -> undefined | keepalive()).
-% start(_, 0, _) ->
-%     undefined;
-% start(StatFun, TimeoutSec, TimeoutMsg) ->
-%     {ok, StatVal} = StatFun(),
-%     #keepalive{statfun = StatFun, statval = StatVal,
-%         tsec = TimeoutSec, tmsg = TimeoutMsg,
-%         tref = timer(TimeoutSec, TimeoutMsg)}.
-
-% %% @doc Check keepalive, called when timeout.
-% -spec(check(keepalive()) -> {ok, keepalive()} | {error, any()}).
-% check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repeat}) ->
-%     case StatFun() of
-%         {ok, NewVal} ->
-%             if NewVal =/= LastVal ->
-%                 {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = 0})};
-%                 Repeat < 1 ->
-%                     {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = Repeat + 1})};
-%                 true ->
-%                     {error, timeout}
-%             end;
-%         {error, Error} ->
-%             {error, Error}
-%     end.
-
-% resume(KeepAlive = #keepalive{tsec = TimeoutSec, tmsg = TimeoutMsg}) ->
-%     KeepAlive#keepalive{tref = timer(TimeoutSec, TimeoutMsg)}.
-
-% %% @doc Cancel Keepalive
-% -spec(cancel(keepalive()) -> ok).
-% cancel(#keepalive{tref = TRef}) ->
-%     cancel(TRef);
-% cancel(undefined) ->
-%     ok;
-% cancel(TRef) ->
-%         catch erlang:cancel_timer(TRef).
-
-% timer(Sec, Msg) ->
-%     erlang:send_after(timer:seconds(Sec), self(), Msg).
-
-
-% log(Format, Args) ->
-%     logger:debug(Format, Args).
diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
new file mode 100644
index 000000000..0e19d9b4f
--- /dev/null
+++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
@@ -0,0 +1,1971 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_lwm2m_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-define(PORT, 5783).
+
+-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)).
+
+-include("src/lwm2m/include/emqx_lwm2m.hrl").
+-include_lib("lwm2m_coap/include/coap.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(CONF_DEFAULT, <<"
+gateway: {
+    lwm2m_xml_dir: \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"
+    lwm2m.1: {
+        lifetime_min: 1s
+        lifetime_max: 86400s
+        qmode_time_windonw: 22
+        auto_observe: false
+        mountpoint: \"lwm2m/%e/\"
+        update_msg_publish_condition: contains_object_list
+        translators: {
+            command: \"dn/#\"
+            response: \"up/resp\"
+            notify: \"up/notify\"
+            register: \"up/resp\"
+            update: \"up/resp\"
+        }
+        listener.udp.1 {
+            bind: 5783
+        }
+    }
+}
+">>).
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+
+all() ->
+    [ {group, test_grp_0_register}
+    , {group, test_grp_1_read}
+    , {group, test_grp_2_write}
+    , {group, test_grp_3_execute}
+    , {group, test_grp_4_discover}
+    , {group, test_grp_5_write_attr}
+    , {group, test_grp_6_observe}
+    , {group, test_grp_8_object_19}
+    ].
+
+suite() -> [{timetrap, {seconds, 90}}].
+
+groups() ->
+    RepeatOpt = {repeat_until_all_ok, 1},
+    [
+        {test_grp_0_register, [RepeatOpt], [
+            case01_register,
+            case01_register_additional_opts,
+            case01_register_incorrect_opts,
+            case01_register_report,
+            case02_update_deregister,
+            case03_register_wrong_version,
+            case04_register_and_lifetime_timeout,
+            case05_register_wrong_epn,
+            case06_register_wrong_lifetime,
+            case07_register_alternate_path_01,
+            case07_register_alternate_path_02,
+            case08_reregister
+        ]},
+        {test_grp_1_read, [RepeatOpt], [
+            case10_read,
+            case10_read_separate_ack,
+            case11_read_object_tlv,
+            case11_read_object_json,
+            case12_read_resource_opaque,
+            case13_read_no_xml
+        ]},
+        {test_grp_2_write, [RepeatOpt], [
+            case20_write,
+            case21_write_object,
+            case22_write_error,
+            case20_single_write
+        ]},
+        {test_grp_create, [RepeatOpt], [
+            case_create_basic
+        ]},
+        {test_grp_delete, [RepeatOpt], [
+            case_delete_basic
+        ]},
+        {test_grp_3_execute, [RepeatOpt], [
+            case30_execute, case31_execute_error
+        ]},
+        {test_grp_4_discover, [RepeatOpt], [
+            case40_discover
+        ]},
+        {test_grp_5_write_attr, [RepeatOpt], [
+            case50_write_attribute
+        ]},
+        {test_grp_6_observe, [RepeatOpt], [
+            case60_observe
+        ]},
+        {test_grp_7_block_wize_transfer, [RepeatOpt], [
+            case70_read_large, case70_write_large
+        ]},
+        {test_grp_8_object_19, [RepeatOpt], [
+            case80_specail_object_19_1_0_write,
+            case80_specail_object_19_0_0_notify
+            %case80_specail_object_19_0_0_response,
+            %case80_normal_object_19_0_0_read
+        ]},
+        {test_grp_9_psm_queue_mode, [RepeatOpt], [
+            case90_psm_mode,
+            case90_queue_mode
+        ]}
+    ].
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx]),
+    Config.
+
+end_per_suite(Config) ->
+    timer:sleep(300),
+    emqx_ct_helpers:stop_apps([emqx]),
+    Config.
+
+init_per_testcase(_AllTestCase, Config) ->
+    ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
+    {ok, _} = application:ensure_all_started(emqx_gateway),
+    {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]),
+
+    {ok, C} = emqtt:start_link([{host, "localhost"},{port, 1883},{clientid, <<"c1">>}]),
+    {ok, _} = emqtt:connect(C),
+    timer:sleep(100),
+
+    [{sock, ClientUdpSock}, {emqx_c, C} | Config].
+
+end_per_testcase(_AllTestCase, Config) ->
+    timer:sleep(300),
+    gen_udp:close(?config(sock, Config)),
+    emqtt:disconnect(?config(emqx_c, Config)),
+    ok = application:stop(emqx_gateway).
+
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+case01_register(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+
+    %% checkpoint 1 - response
+    #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
+        test_recv_coap_response(UdpSock),
+    ack = Type,
+    {ok, created} = Method,
+    RspId = MsgId,
+    Location = proplists:get_value(location_path, Opts),
+    ?assertNotEqual(undefined, Location),
+
+    %% checkpoint 2 - verify subscribed topics
+    timer:sleep(50),
+    ?LOGT("all topics: ~p", [test_mqtt_broker:get_subscrbied_topics()]),
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
+
+    % ----------------------------------------
+    % DE-REGISTER command
+    % ----------------------------------------
+    ?LOGT("start to send DE-REGISTER command", []),
+    MsgId3 = 52,
+    test_send_coap_request( UdpSock,
+                            delete,
+                            sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{payload = <<>>},
+                            [],
+                            MsgId3),
+    #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
+    {ok,deleted} = Method3,
+    MsgId3 = RspId3,
+    timer:sleep(50),
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case01_register_additional_opts(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+
+    AddOpts = "ep=~s&lt=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2",
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+
+    %% checkpoint 1 - response
+    #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
+        test_recv_coap_response(UdpSock),
+    Type = ack,
+    Method = {ok, created},
+    RspId = MsgId,
+    Location = proplists:get_value(location_path, Opts),
+    ?assertNotEqual(undefined, Location),
+
+    %% checkpoint 2 - verify subscribed topics
+    timer:sleep(50),
+
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
+
+    % ----------------------------------------
+    % DE-REGISTER command
+    % ----------------------------------------
+    ?LOGT("start to send DE-REGISTER command", []),
+    MsgId3 = 52,
+    test_send_coap_request( UdpSock,
+                            delete,
+                            sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{payload = <<>>},
+                            [],
+                            MsgId3),
+    #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
+    {ok,deleted} = Method3,
+    MsgId3 = RspId3,
+    timer:sleep(50),
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case01_register_incorrect_opts(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+
+
+    AddOpts = "ep=~s&lt=345&lwm2m=1&incorrect_opt",
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+
+    %% checkpoint 1 - response
+    #coap_message{type = ack, method = Method, id = MsgId} =
+        test_recv_coap_response(UdpSock),
+    ?assertEqual({error,bad_request}, Method).
+
+case01_register_report(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+
+    #coap_message{type = Type, method = Method, id = RspId, options = Opts} =
+        test_recv_coap_response(UdpSock),
+    Type = ack,
+    Method = {ok, created},
+    RspId = MsgId,
+    Location = proplists:get_value(location_path, Opts),
+    ?assertNotEqual(undefined, Location),
+
+    timer:sleep(50),
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"msgType">> => <<"register">>,
+                                <<"data">> => #{
+                                    <<"alternatePath">> => <<"/">>,
+                                    <<"ep">> => list_to_binary(Epn),
+                                    <<"lt">> => 345,
+                                    <<"lwm2m">> => <<"1">>,
+                                    <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)),
+
+    % ----------------------------------------
+    % DE-REGISTER command
+    % ----------------------------------------
+    ?LOGT("start to send DE-REGISTER command", []),
+    MsgId3 = 52,
+    test_send_coap_request( UdpSock,
+                            delete,
+                            sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{payload = <<>>},
+                            [],
+                            MsgId3),
+    #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
+    {ok,deleted} = Method3,
+    MsgId3 = RspId3,
+    timer:sleep(50),
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case02_update_deregister(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+    timer:sleep(100),
+    #coap_message{type = ack, method = Method, options = Opts} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method),
+
+    ?LOGT("Options got: ~p", [Opts]),
+    Location = proplists:get_value(location_path, Opts),
+    Register = emqx_json:encode(#{
+                              <<"msgType">> => <<"register">>,
+                              <<"data">> => #{
+                                  <<"alternatePath">> => <<"/">>,
+                                  <<"ep">> => list_to_binary(Epn),
+                                  <<"lt">> => 345,
+                                  <<"lwm2m">> => <<"1">>,
+                                  <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>]
+                              }
+                            }),
+    ?assertEqual(Register, test_recv_mqtt_response(ReportTopic)),
+
+    % ----------------------------------------
+    % UPDATE command
+    % ----------------------------------------
+    ?LOGT("start to send UPDATE command", []),
+    MsgId2 = 27,
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>, </6>">>},
+                            [],
+                            MsgId2),
+    #coap_message{type = ack, id = RspId2, method = Method2} = test_recv_coap_response(UdpSock),
+    {ok,changed} = Method2,
+    MsgId2 = RspId2,
+    Update = emqx_json:encode(#{
+                            <<"msgType">> => <<"update">>,
+                            <<"data">> => #{
+                                <<"alternatePath">> => <<"/">>,
+                                <<"ep">> => list_to_binary(Epn),
+                                <<"lt">> => 789,
+                                <<"lwm2m">> => <<"1">>,
+                                <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>, <<"/6">>]
+                            }
+                        }),
+    ?assertEqual(Update, test_recv_mqtt_response(ReportTopic)),
+
+    % ----------------------------------------
+    % DE-REGISTER command
+    % ----------------------------------------
+    ?LOGT("start to send DE-REGISTER command", []),
+    MsgId3 = 52,
+    test_send_coap_request( UdpSock,
+                            delete,
+                            sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{payload = <<>>},
+                            [],
+                            MsgId3),
+    #coap_message{type = ack, id = RspId3, method = Method3} = test_recv_coap_response(UdpSock),
+    {ok,deleted} = Method3,
+    MsgId3 = RspId3,
+
+    timer:sleep(50),
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case03_register_wrong_version(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=8.3", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+    #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
+    ?assertEqual({error,precondition_failed}, Method),
+    timer:sleep(50),
+
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case04_register_and_lifetime_timeout(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=2&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+    timer:sleep(100),
+    #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method),
+
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
+
+    % ----------------------------------------
+    % lifetime timeout
+    % ----------------------------------------
+    timer:sleep(4000),
+
+    false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case05_register_wrong_epn(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    MsgId = 12,
+    UdpSock = ?config(sock, Config),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?lt=345&lwm2m=1.0", [?PORT]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+    #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
+    ?assertEqual({error,bad_request}, Method).
+
+case06_register_wrong_lifetime(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId),
+    #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock),
+    ?assertEqual({error,bad_request}, Method),
+    timer:sleep(50),
+    ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()).
+
+case07_register_alternate_path_01(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId),
+    timer:sleep(50),
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case07_register_alternate_path_02(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId),
+    timer:sleep(50),
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
+
+case08_reregister(Config) ->
+    % ----------------------------------------
+    % REGISTER command
+    % ----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+    SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"),
+    ReportTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), ReportTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId),
+    timer:sleep(50),
+    true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()),
+
+    ReadResult = emqx_json:encode(
+                            #{
+                                <<"msgType">> => <<"register">>,
+                                <<"data">> => #{
+                                    <<"alternatePath">> => <<"/lwm2m">>,
+                                    <<"ep">> => list_to_binary(Epn),
+                                    <<"lt">> => 345,
+                                    <<"lwm2m">> => <<"1">>,
+                                    <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>]
+                                 }
+                             }
+                            ),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)),
+    timer:sleep(1000),
+
+    %% the same lwm2mc client registers to server again
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId + 1),
+    %% verify the lwm2m client is still online
+    ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)).
+
+case10_read(Config) ->
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+    % step 1, device register ...
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId1),
+    #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method1),
+    test_recv_mqtt_response(RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<"/lwm2m/3/0/0">>, get_coap_path(Options2)),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"reqPath">> => <<"/3/0/0">>,
+                                    <<"content">> => [#{
+                                        path => <<"/3/0/0">>,
+                                        value => <<"EMQ">>
+                                    }]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case10_read_separate_ack(Config) ->
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    % step 1, device register ...
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command = #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<"/3/0/0">>, get_coap_path(Options2)),
+    ?assertEqual(<<>>, Payload2),
+
+    test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2),
+    ReadResultACK = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"ack">>,
+                                <<"data">> => #{
+                                    <<"path">> => <<"/3/0/0">>
+                                }
+                             }),
+    ?assertEqual(ReadResultACK, test_recv_mqtt_response(RespTopic)),
+    timer:sleep(100),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, false),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"reqPath">> => <<"/3/0/0">>,
+                                    <<"content">> => [#{
+                                        path => <<"/3/0/0">>,
+                                        value => <<"EMQ">>
+                                    }]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case11_read_object_tlv(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 207,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    timer:sleep(50),
+
+    Tlv = <<16#08, 16#00, 16#3C, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>,
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+tlv">>, payload = Tlv}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"reqPath">> => <<"/3/0">>,
+                                    <<"content">> => [
+                                        #{
+                                            path => <<"/3/0/0">>,
+                                            value => <<"Open Mobile Alliance">>
+                                        },
+                                        #{
+                                            path => <<"/3/0/1">>,
+                                            value => <<"Lightweight M2M Client">>
+                                        },
+                                        #{
+                                            path => <<"/3/0/2">>,
+                                            value => <<"345000123">>
+                                        }
+                                    ]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case11_read_object_json(Config) ->
+    % step 1, device register ...
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    timer:sleep(50),
+
+    Json = <<"{\"bn\":\"/3/0\",\"e\":[{\"n\":\"0\",\"sv\":\"Open Mobile Alliance\"},{\"n\":\"1\",\"sv\":\"Lightweight M2M Client\"},{\"n\":\"2\",\"sv\":\"345000123\"}]}">>,
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/vnd.oma.lwm2m+json">>, payload = Json}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"reqPath">> => <<"/3/0">>,
+                                    <<"content">> => [
+                                        #{
+                                            path => <<"/3/0/0">>,
+                                            value => <<"Open Mobile Alliance">>
+                                        },
+                                        #{
+                                            path => <<"/3/0/1">>,
+                                            value => <<"Lightweight M2M Client">>
+                                        },
+                                        #{
+                                            path => <<"/3/0/2">>,
+                                            value => <<"345000123">>
+                                        }
+                                    ]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case12_read_resource_opaque(Config) ->
+    % step 1, device register ...
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0/8">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    timer:sleep(50),
+
+    Opaque = <<20, 21, 22, 23>>,
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"application/octet-stream">>, payload = Opaque}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"reqPath">> => <<"/3/0/8">>,
+                                    <<"content">> => [
+                                        #{
+                                            path => <<"/3/0/8">>,
+                                            value => base64:encode(Opaque)
+                                        }
+                                    ]
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case13_read_no_xml(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/9723/0/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+
+    ?assertEqual(get, Method2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/9723/0/0">>,
+                                    <<"code">> => <<"4.00">>,
+                                    <<"codeMsg">> => <<"bad_request">>
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case20_single_write(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/13">>,
+                    <<"type">> => <<"Integer">>,
+                    <<"value">> => <<"12345">>
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(put, Method2),
+    ?assertEqual(<<"/3/0/13">>, Path2),
+    Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16>>,
+    ?assertEqual(Tlv_Value, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/13">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                },
+                                <<"msgType">> => <<"write">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case20_write(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write">>,
+                <<"data">> => #{
+                    <<"basePath">> => <<"/3/0/13">>,
+                    <<"content">> => [#{
+                        type => <<"Float">>,
+                        value => <<"12345.0">>
+                    }]
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(put, Method2),
+    ?assertEqual(<<"/3/0/13">>, Path2),
+    Tlv_Value = <<200, 13, 8, 64,200,28,128,0,0,0,0>>,
+    ?assertEqual(Tlv_Value, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    WriteResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/13">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                },
+                                <<"msgType">> => <<"write">>
+                            }),
+    ?assertEqual(WriteResult, test_recv_mqtt_response(RespTopic)).
+
+case21_write_object(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write">>,
+                <<"data">> => #{
+                    <<"basePath">> => <<"/3/0/">>,
+                    <<"content">> => [#{
+                        path => <<"13">>,
+                        type => <<"Integer">>,
+                        value => <<"12345">>
+                    },#{
+                        path => <<"14">>,
+                        type => <<"String">>,
+                        value => <<"87x">>
+                    }]
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(post, Method2),
+    ?assertEqual(<<"/3/0">>, Path2),
+    Tlv_Value = <<3:2, 0:1, 0:2, 2:3, 13, 12345:16,
+                  3:2, 0:1, 0:2, 3:3, 14, "87x">>,
+    ?assertEqual(Tlv_Value, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"write">>,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                }
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case22_write_error(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write">>,
+                <<"data">> => #{
+                    <<"basePath">> => <<"/3/0/1">>,
+                    <<"content">> => [
+                        #{
+                            type => <<"Integer">>,
+                            value => <<"12345">>
+                        }
+                    ]
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(put, Method2),
+    ?assertEqual(<<"/3/0/1">>, Path2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, bad_request}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/1">>,
+                                    <<"code">> => <<"4.00">>,
+                                    <<"codeMsg">> => <<"bad_request">>
+                                },
+                                <<"msgType">> => <<"write">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case_create_basic(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2, send a CREATE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"create">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/5">>
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(post, Method2),
+    ?assertEqual(<<"/5">>, Path2),
+    ?assertEqual(<<"">>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, created}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/5">>,
+                                    <<"code">> => <<"2.01">>,
+                                    <<"codeMsg">> => <<"created">>
+                                },
+                                <<"msgType">> => <<"create">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case_delete_basic(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>, </5/0>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2, send a CREATE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"delete">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/5/0">>
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(delete, Method2),
+    ?assertEqual(<<"/5/0">>, Path2),
+    ?assertEqual(<<"">>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, deleted}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/5/0">>,
+                                    <<"code">> => <<"2.02">>,
+                                    <<"codeMsg">> => <<"deleted">>
+                                },
+                                <<"msgType">> => <<"delete">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case30_execute(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"execute">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/4">>,
+                    %% "args" should not be present for "/3/0/4", only for testing the encoding here
+                    <<"args">> => <<"2,7">>
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(post, Method2),
+    ?assertEqual(<<"/3/0/4">>, Path2),
+    ?assertEqual(<<"2,7">>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/4">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                },
+                                <<"msgType">> => <<"execute">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case31_execute_error(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"execute">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/4">>,
+                    <<"args">> => <<"2,7">>
+                }
+               },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(post, Method2),
+    ?assertEqual(<<"/3/0/4">>, Path2),
+    ?assertEqual(<<"2,7">>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, uauthorized}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/4">>,
+                                    <<"code">> => <<"4.01">>,
+                                    <<"codeMsg">> => <<"uauthorized">>
+                                },
+                                <<"msgType">> => <<"execute">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case40_discover(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"discover">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/7">>
+                } },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<"/3/0/7">>, Path2),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    PayloadDiscover = <<"</3/0/7>;dim=8;pmin=10;pmax=60;gt=50;lt=42.2,</3/0/8>">>,
+    test_send_coap_response(UdpSock,
+                            "127.0.0.1",
+                            ?PORT,
+                            {ok, content},
+                            #coap_content{content_format = <<"application/link-format">>, payload = PayloadDiscover},
+                            Request2,
+                            true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"discover">>,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/7">>,
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"content">> =>
+                                        [<<"</3/0/7>;dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"</3/0/8>">>]
+                                }
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case50_write_attribute(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write-attr">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/9">>,
+                    <<"pmin">> => <<"1">>,
+                    <<"pmax">> => <<"5">>,
+                    <<"lt">> => <<"5">>
+                } },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(100),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    ?LOGT("got options: ~p", [Options2]),
+    Path2 = get_coap_path(Options2),
+    Query2 = lists:sort(get_coap_query(Options2)),
+    ?assertEqual(put, Method2),
+    ?assertEqual(<<"/3/0/9">>, Path2),
+    ?assertEqual(lists:sort([<<"pmax=5">>,<<"lt=5">>,<<"pmin=1">>]), Query2),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock,
+                            "127.0.0.1",
+                            ?PORT,
+                            {ok, changed},
+                            #coap_content{},
+                            Request2,
+                            true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/9">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                },
+                                <<"msgType">> => <<"write-attr">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case60_observe(Config) ->
+    % step 1, device register ...
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    ObjectList = <<"</1>, </2>, </3/0>, </4>, </5>">>,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    RespTopicAD = list_to_binary("lwm2m/"++Epn++"/up/notify"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopicAD, qos0),
+    timer:sleep(200),
+
+    std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
+
+    % step2,  send a OBSERVE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"observe">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/10">>
+                }
+             },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    Observe = get_coap_observe(Options2),
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<"/3/0/10">>, Path2),
+    ?assertEqual(Observe, 0),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_observe_ack( UdpSock,
+                                "127.0.0.1",
+                                ?PORT,
+                                {ok, content},
+                                #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>},
+                                Request2),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"observe">>,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/10">>,
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"content">> => [#{
+                                        path => <<"/3/0/10">>,
+                                        value => 2048
+                                    }]
+                                }
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)),
+
+    %% step3 the notifications
+    timer:sleep(200),
+    ObSeq = 3,
+    test_send_coap_notif(   UdpSock,
+                            "127.0.0.1",
+                            ?PORT,
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"4096">>},
+                            ObSeq,
+                            Request2),
+    timer:sleep(100),
+    #coap_message{} = test_recv_coap_response(UdpSock),
+
+    ReadResult2 = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"notify">>,
+                                <<"seqNum">> => ObSeq,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/10">>,
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"content">> => [#{
+                                        path => <<"/3/0/10">>,
+                                        value => 4096
+                                    }]
+                                }
+                            }),
+    ?assertEqual(ReadResult2, test_recv_mqtt_response(RespTopicAD)),
+
+    %% Step3. cancel observe
+    CmdId3 = 308,
+    Command3 = #{<<"requestID">> => CmdId3, <<"cacheID">> => CmdId3,
+                <<"msgType">> => <<"cancel-observe">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/3/0/10">>
+                }
+             },
+    CommandJson3 = emqx_json:encode(Command3),
+    test_mqtt_broker:publish(CommandTopic, CommandJson3, 0),
+    timer:sleep(50),
+    Request3 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method3, options=Options3, payload=Payload3} = Request3,
+    Path3 = get_coap_path(Options3),
+    Observe3 = get_coap_observe(Options3),
+    ?assertEqual(get, Method3),
+    ?assertEqual(<<"/3/0/10">>, Path3),
+    ?assertEqual(Observe3, 1),
+    ?assertEqual(<<>>, Payload3),
+    timer:sleep(50),
+
+    test_send_coap_observe_ack( UdpSock,
+                                "127.0.0.1",
+                                ?PORT,
+                                {ok, content},
+                                #coap_content{content_format = <<"text/plain">>, payload = <<"1150">>},
+                                Request3),
+    timer:sleep(100),
+
+    ReadResult3 = emqx_json:encode(#{
+                                <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3,
+                                <<"msgType">> => <<"cancel-observe">>,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/3/0/10">>,
+                                    <<"code">> => <<"2.05">>,
+                                    <<"codeMsg">> => <<"content">>,
+                                    <<"content">> => [#{
+                                        path => <<"/3/0/10">>,
+                                        value => 1150
+                                    }]
+                                }
+                            }),
+    ?assertEqual(ReadResult3, test_recv_mqtt_response(RespTopic)).
+
+case80_specail_object_19_0_0_notify(Config) ->
+    % step 1, device register, with extra register options
+    Epn = "urn:oma:lwm2m:oma:3",
+    RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId1),
+    #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method1),
+    ReadResult = emqx_json:encode(#{
+                                <<"msgType">> => <<"register">>,
+                                <<"data">> => #{
+                                    <<"alternatePath">> => <<"/">>,
+                                    <<"ep">> => list_to_binary(Epn),
+                                    <<"lt">> => 345,
+                                    <<"lwm2m">> => <<"1">>,
+                                    <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>],
+                                    <<"apn">> => <<"psmA.eDRX0.ctnb">>,
+                                    <<"im">> => <<"13456">>,
+                                    <<"ct">> => <<"2.0">>,
+                                    <<"mt">> => <<"MDM9206">>,
+                                    <<"mv">> => <<"4.0">>
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)),
+
+    % step2,  send a OBSERVE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"observe">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/19/0/0">>
+                }
+             },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    Observe = get_coap_observe(Options2),
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<"/19/0/0">>, Path2),
+    ?assertEqual(Observe, 0),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_observe_ack( UdpSock,
+                                "127.0.0.1",
+                                ?PORT,
+                                {ok, content},
+                                #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>},
+                                Request2),
+    timer:sleep(100).
+
+    %% step 3, device send uplink data notifications
+
+case80_specail_object_19_1_0_write(Config) ->
+    Epn = "urn:oma:lwm2m:oma:3",
+    RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0",
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId1),
+    #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method1),
+    test_recv_mqtt_response(RespTopic),
+
+    % step2,  send a WRITE command to device
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    CmdId = 307,
+    Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                <<"msgType">> => <<"write">>,
+                <<"data">> => #{
+                    <<"path">> => <<"/19/1/0">>,
+                    <<"type">> => <<"Opaque">>,
+                    <<"value">> => base64:encode(<<12345:32>>)
+                }
+               },
+
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2,
+    Path2 = get_coap_path(Options2),
+    ?assertEqual(put, Method2),
+    ?assertEqual(<<"/19/1/0">>, Path2),
+    ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{
+                                <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"data">> => #{
+                                    <<"reqPath">> => <<"/19/1/0">>,
+                                    <<"code">> => <<"2.04">>,
+                                    <<"codeMsg">> => <<"changed">>
+                                },
+                                <<"msgType">> => <<"write">>
+                            }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+case90_psm_mode(Config) ->
+    server_cache_mode(Config, "ep=~s&lt=345&lwm2m=1&apn=psmA.eDRX0.ctnb").
+
+case90_queue_mode(Config) ->
+    server_cache_mode(Config, "ep=~s&lt=345&lwm2m=1&b=UQ").
+
+server_cache_mode(Config, RegOption) ->
+    application:set_env(?APP, qmode_time_window, 2),
+
+    % step 1, device register, with apn indicates "PSM" mode
+    Epn = "urn:oma:lwm2m:oma:3",
+
+    MsgId1 = 15,
+    UdpSock = ?config(sock, Config),
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?"++RegOption, [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = <<"</1>, </2>, </3>, </4>, </5>">>},
+                            [],
+                            MsgId1),
+    #coap_message{type = ack, method = Method1, options = Opts} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method1),
+    ?LOGT("Options got: ~p", [Opts]),
+    Location = proplists:get_value(location_path, Opts),
+    test_recv_mqtt_response(RespTopic),
+
+    %% server not in PSM mode
+    send_read_command_1(0, UdpSock),
+    verify_read_response_1(0, UdpSock),
+
+    %% server inters into PSM mode
+    timer:sleep(2),
+
+    %% verify server caches downlink commands
+    send_read_command_1(1, UdpSock),
+    send_read_command_1(2, UdpSock),
+    send_read_command_1(3, UdpSock),
+
+    ?assertEqual(timeout_test_recv_coap_request, test_recv_coap_request(UdpSock)),
+
+    device_update_1(UdpSock, Location),
+
+    verify_read_response_1(1, UdpSock),
+    verify_read_response_1(2, UdpSock),
+    verify_read_response_1(3, UdpSock).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% Internal Functions
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+send_read_command_1(CmdId, _UdpSock) ->
+    Epn = "urn:oma:lwm2m:oma:3",
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3/0/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50).
+
+verify_read_response_1(CmdId, UdpSock) ->
+    Epn = "urn:oma:lwm2m:oma:3",
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+
+    %% device receives a command
+    Request = test_recv_coap_request(UdpSock),
+    ?LOGT("LwM2M client got ~p", [Request]),
+
+    %% device replies the commond
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request, true),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                            <<"msgType">> => <<"read">>,
+                            <<"data">> => #{
+                                <<"code">> => <<"2.05">>,
+                                <<"codeMsg">> => <<"content">>,
+                                <<"content">> => [#{
+                                    path => <<"/3/0/0">>,
+                                    value => <<"EMQ">>
+                                }]
+                            }
+                         }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+device_update_1(UdpSock, Location) ->
+    Epn = "urn:oma:lwm2m:oma:3",
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    ?LOGT("send UPDATE command", []),
+    MsgId2 = 27,
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]),
+                            #coap_content{payload = <<>>},
+                            [],
+                            MsgId2),
+    #coap_message{type = ack, id = MsgId2, method = Method2} = test_recv_coap_response(UdpSock),
+    {ok,changed} = Method2,
+    test_recv_mqtt_response(RespTopic).
+
+test_recv_mqtt_response(RespTopic) ->
+    receive
+        {publish, #{topic := RespTopic, payload := RM}} ->
+            ?LOGT("test_recv_mqtt_response Response=~p", [RM]),
+            RM
+    after 1000 -> timeout_test_recv_mqtt_response
+    end.
+
+test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) ->
+    is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
+    is_list(Options) orelse error("Options must be a list"),
+    case resolve_uri(Uri) of
+        {coap, {IpAddr, Port}, Path, Query} ->
+            Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]),
+            Request = Request0#coap_message{id = MsgId},
+            ?LOGT("send_coap_request Request=~p", [Request]),
+            RequestBinary = lwm2m_coap_message_parser:encode(Request),
+            ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]),
+            ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary);
+        {SchemeDiff, ChIdDiff, _, _} ->
+            error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff])))
+    end.
+
+test_recv_coap_response(UdpSock) ->
+    {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000),
+    Response = lwm2m_coap_message_parser:decode(Packet),
+    ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]),
+    #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response,
+    ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]),
+    Response.
+
+test_recv_coap_request(UdpSock) ->
+    case gen_udp:recv(UdpSock, 0, 2000) of
+        {ok, {_Address, _Port, Packet}} ->
+            Request = lwm2m_coap_message_parser:decode(Packet),
+            #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request,
+            ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]),
+            Request;
+        {error, Reason} ->
+            ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]),
+            timeout_test_recv_coap_request
+    end.
+
+test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) ->
+    is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
+    is_list(Host) orelse error("Host is not a string"),
+
+    {ok, IpAddr} = inet:getaddr(Host, inet),
+    Response = lwm2m_coap_message:response(Code, Content, Request),
+    Response2 = case Ack of
+                    true -> Response#coap_message{type = ack};
+                    false -> Response
+                end,
+    ?LOGT("test_send_coap_response Response=~p", [Response2]),
+    ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)).
+
+test_send_empty_ack(UdpSock, Host, Port, Request) ->
+    is_list(Host) orelse error("Host is not a string"),
+    {ok, IpAddr} = inet:getaddr(Host, inet),
+    EmptyACK = lwm2m_coap_message:ack(Request),
+    ?LOGT("test_send_empty_ack EmptyACK=~p", [EmptyACK]),
+    ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(EmptyACK)).
+
+test_send_coap_observe_ack(UdpSock, Host, Port, Code, Content, Request) ->
+    is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
+    is_list(Host) orelse error("Host is not a string"),
+
+    {ok, IpAddr} = inet:getaddr(Host, inet),
+    Response = lwm2m_coap_message:response(Code, Content, Request),
+    Response1 = lwm2m_coap_message:set(observe, 0, Response),
+    Response2 = Response1#coap_message{type = ack},
+
+    ?LOGT("test_send_coap_observe_ack Response=~p", [Response2]),
+    ResponseBinary = lwm2m_coap_message_parser:encode(Response2),
+    ok = gen_udp:send(UdpSock, IpAddr, Port, ResponseBinary).
+
+test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) ->
+    is_record(Content, coap_content) orelse error("Content must be a #coap_content!"),
+    is_list(Host) orelse error("Host is not a string"),
+
+    {ok, IpAddr} = inet:getaddr(Host, inet),
+    Notif = lwm2m_coap_message:response({ok, content}, Content, Request),
+    NewNotif = lwm2m_coap_message:set(observe, ObSeq, Notif),
+    ?LOGT("test_send_coap_notif Response=~p", [NewNotif]),
+    NotifBinary = lwm2m_coap_message_parser:encode(NewNotif),
+    ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, NotifBinary]),
+    ok = gen_udp:send(UdpSock, IpAddr, Port, NotifBinary).
+
+std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) ->
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>, payload = ObjectList},
+                            [],
+                            MsgId1),
+    #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock),
+    test_recv_mqtt_response(RespTopic),
+    timer:sleep(100).
+
+resolve_uri(Uri) ->
+    {ok, #{scheme := Scheme,
+           host := Host,
+           port := PortNo,
+           path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri),
+    Query = maps:get(query, URIMap, ""),
+    {ok, PeerIP} = inet:getaddr(Host, inet),
+    {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}.
+
+split_path([]) -> [];
+split_path([$/]) -> [];
+split_path([$/ | Path]) -> split_segments(Path, $/, []).
+
+split_query([]) -> [];
+split_query(Path) -> split_segments(Path, $&, []).
+
+split_segments(Path, Char, Acc) ->
+    case string:rchr(Path, Char) of
+        0 ->
+            [make_segment(Path) | Acc];
+        N when N > 0 ->
+            split_segments(string:substr(Path, 1, N-1), Char,
+                [make_segment(string:substr(Path, N+1)) | Acc])
+    end.
+
+make_segment(Seg) ->
+    list_to_binary(emqx_http_lib:uri_decode(Seg)).
+
+
+get_coap_path(Options) ->
+    get_path(Options, <<>>).
+
+get_coap_query(Options) ->
+    proplists:get_value(uri_query, Options, []).
+
+get_coap_observe(Options) ->
+    get_observe(Options).
+
+
+get_path([], Acc) ->
+    %?LOGT("get_path Acc=~p", [Acc]),
+    Acc;
+get_path([{uri_path, Path1}|T], Acc) ->
+    %?LOGT("Path=~p, Acc=~p", [Path1, Acc]),
+    get_path(T, join_path(Path1, Acc));
+get_path([{_, _}|T], Acc) ->
+    get_path(T, Acc).
+
+get_observe([]) ->
+    undefined;
+get_observe([{observe, V}|_T]) ->
+    V;
+get_observe([{_, _}|T]) ->
+    get_observe(T).
+
+join_path([], Acc) -> Acc;
+join_path([<<"/">>|T], Acc) ->
+    join_path(T, Acc);
+join_path([H|T], Acc) ->
+    join_path(T, <<Acc/binary, $/, H/binary>>).
+
+sprintf(Format, Args) ->
+    lists:flatten(io_lib:format(Format, Args)).
diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl
new file mode 100644
index 000000000..fd62ccf9d
--- /dev/null
+++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl
@@ -0,0 +1,238 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_tlv_SUITE).
+
+ -compile(export_all).
+ -compile(nowarn_export_all).
+
+ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)).
+
+ -include("src/lwm2m/include/emqx_lwm2m.hrl").
+ -include_lib("lwm2m_coap/include/coap.hrl").
+ -include_lib("eunit/include/eunit.hrl").
+
+
+ all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09].
+
+ init_per_suite(Config) ->
+     Config.
+
+ end_per_suite(Config) ->
+     Config.
+
+
+ case01(_Config) ->
+     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case02(_Config) ->
+     Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_multiple_resource => 16#06, value => [
+                                                                     #{tlv_resource_instance => 16#00, value => <<1>>},
+                                                                     #{tlv_resource_instance => 16#01, value => <<5>>}
+             ]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case03(_Config) ->
+     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
+             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
+             #{tlv_resource_with_value => 16#02, value => <<"345000123">>}
+             ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case03_0(_Config) ->
+     Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_multiple_resource => 16#02, value => [
+             #{tlv_resource_instance => 16#7F, value => <<16#07>>},
+             #{tlv_resource_instance => 16#0136, value => <<16#01>>}
+         ]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case04(_Config) ->
+     % 6.4.3.1 Single Object Instance Request Example
+     Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
+             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
+             #{tlv_resource_with_value => 16#02, value => <<"345000123">>},
+             #{tlv_resource_with_value => 16#03, value => <<"1.0">>},
+             #{tlv_multiple_resource => 16#06, value => [
+                                                                         #{tlv_resource_instance => 16#00, value => <<1>>},
+                                                                         #{tlv_resource_instance => 16#01, value => <<5>>}
+                 ]},
+             #{tlv_multiple_resource => 16#07, value => [
+                                                                         #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>},
+                                                                         #{tlv_resource_instance => 16#01, value => <<16#1388:16>>}
+                 ]},
+             #{tlv_multiple_resource => 16#08, value => [
+                                                                         #{tlv_resource_instance => 16#00, value => <<16#7d>>},
+                                                                         #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}
+             ]},
+             #{tlv_resource_with_value => 16#09, value => <<16#64>>},
+             #{tlv_resource_with_value => 16#0A, value => <<16#0F>>},
+             #{tlv_multiple_resource => 16#0B, value => [
+                 #{tlv_resource_instance => 16#00, value => <<16#00>>}
+             ]},
+             #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>},
+             #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>},
+             #{tlv_resource_with_value => 16#10, value => <<"U">>}
+           ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case05(_Config) ->
+     % 6.4.3.2 Multiple Object Instance Request Examples
+     %   A) Request on Single-Instance Object
+     Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_object_instance => 16#00, value => [
+             #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>},
+             #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>},
+             #{tlv_resource_with_value => 16#02, value => <<"345000123">>},
+             #{tlv_resource_with_value => 16#03, value => <<"1.0">>},
+             #{tlv_multiple_resource => 16#06, value => [
+                 #{tlv_resource_instance => 16#00, value => <<1>>},
+                 #{tlv_resource_instance => 16#01, value => <<5>>}
+             ]},
+             #{tlv_multiple_resource => 16#07, value => [
+                 #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>},
+                 #{tlv_resource_instance => 16#01, value => <<16#1388:16>>}
+             ]},
+             #{tlv_multiple_resource => 16#08, value => [
+                 #{tlv_resource_instance => 16#00, value => <<16#7d>>},
+                 #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}
+             ]},
+             #{tlv_resource_with_value => 16#09, value => <<16#64>>},
+             #{tlv_resource_with_value => 16#0A, value => <<16#0F>>},
+             #{tlv_multiple_resource => 16#0B, value => [
+                 #{tlv_resource_instance => 16#00, value => <<16#00>>}
+             ]},
+             #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>},
+             #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>},
+             #{tlv_resource_with_value => 16#10, value => <<"U">>}
+         ]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case06(_Config) ->
+     % 6.4.3.2 Multiple Object Instance Request Examples
+     %   B) Request on Multiple-Instances Object having 2 instances
+     Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_object_instance => 16#00, value => [
+             #{tlv_resource_with_value => 16#00, value => <<16#01>>},
+             #{tlv_resource_with_value => 16#01, value => <<16#00>>},
+             #{tlv_multiple_resource => 16#02, value => [
+                 #{tlv_resource_instance => 16#7F, value => <<16#07>>}
+             ]},
+             #{tlv_resource_with_value => 16#03, value => <<16#7F>>}
+         ]},
+         #{tlv_object_instance => 16#02, value => [
+             #{tlv_resource_with_value => 16#00, value => <<16#03>>},
+             #{tlv_resource_with_value => 16#01, value => <<16#00>>},
+             #{tlv_multiple_resource => 16#02, value => [
+                 #{tlv_resource_instance => 16#7F, value => <<16#07>>},
+                 #{tlv_resource_instance => 16#0136, value => <<16#01>>}
+             ]},
+             #{tlv_resource_with_value => 16#03, value => <<16#7F>>}
+         ]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case07(_Config) ->
+     % 6.4.3.2 Multiple Object Instance Request Examples
+     %   C) Request on Multiple-Instances Object having 1 instance only
+     Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_object_instance => 16#00, value => [
+             #{tlv_resource_with_value => 16#00, value => <<16#01>>},
+             #{tlv_resource_with_value => 16#01, value => <<86400:32>>},
+             #{tlv_resource_with_value => 16#06, value => <<16#01>>},
+             #{tlv_resource_with_value => 16#07, value => <<$U>>}]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case08(_Config) ->
+     % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource
+     %   Example 1) request to Object 65 Instance 0: Read /65/0
+     Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_multiple_resource => 16#00, value => [
+             #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>},
+             #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>}
+             ]},
+         #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>},
+         #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
+ case09(_Config) ->
+     % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource
+     %   Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances
+     Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>,
+     R = emqx_lwm2m_tlv:parse(Data),
+     Exp = [
+         #{tlv_object_instance => 16#00, value => [
+             #{tlv_resource_with_value => 16#00, value => <<"myService 1">>},
+             #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>},
+             #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>}
+         ]},
+         #{tlv_object_instance => 16#01, value => [
+             #{tlv_resource_with_value => 16#00, value => <<"myService 2">>},
+             #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>},
+             #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>}
+         ]}
+     ],
+     ?assertEqual(Exp, R),
+     EncodedBinary = emqx_lwm2m_tlv:encode(Exp),
+     ?assertEqual(EncodedBinary, Data).
+
diff --git a/apps/emqx_gateway/test/test_mqtt_broker.erl b/apps/emqx_gateway/test/test_mqtt_broker.erl
new file mode 100644
index 000000000..dd85340b6
--- /dev/null
+++ b/apps/emqx_gateway/test/test_mqtt_broker.erl
@@ -0,0 +1,171 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(test_mqtt_broker).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-define(LOGT(Format, Args), logger:debug("TEST_BROKER: " ++ Format, Args)).
+
+-record(state, {subscriber}).
+
+-include_lib("emqx/include/emqx.hrl").
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+
+-include_lib("eunit/include/eunit.hrl").
+
+start(_, <<"attacker">>, _, _, _) ->
+    {stop, auth_failure};
+start(ClientId, Username, Password, _Channel, KeepaliveInterval) ->
+    true = is_binary(ClientId),
+    (true = ( is_binary(Username)) orelse (Username == undefined) ),
+    (true = ( is_binary(Password)) orelse (Password == undefined) ),
+    self() ! {keepalive, start, KeepaliveInterval},
+    {ok, []}.
+
+publish(Topic, Payload, Qos) ->
+    ClientId = <<"lwm2m_test_suite">>,
+    Msg = emqx_message:make(ClientId, Qos, Topic, Payload),
+    emqx:publish(Msg).
+
+subscribe(Topic) ->
+    gen_server:call(?MODULE, {subscribe, Topic, self()}).
+
+unsubscribe(Topic) ->
+    gen_server:call(?MODULE, {unsubscribe, Topic}).
+
+get_subscrbied_topics() ->
+    [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)].
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+stop() ->
+    gen_server:stop(?MODULE).
+
+init(_Param) ->
+    {ok, #state{subscriber = []}}.
+
+handle_call({subscribe, Topic, Proc}, _From, State=#state{subscriber = SubList}) ->
+    ?LOGT("test broker subscribe Topic=~p, Pid=~p~n", [Topic, Proc]),
+    is_binary(Topic) orelse error("Topic should be a binary"),
+    {reply, {ok, []}, State#state{subscriber = [{Topic, Proc}|SubList]}};
+
+handle_call(get_subscribed_topics, _From, State=#state{subscriber = SubList}) ->
+    Response = subscribed_topics(SubList, []),
+    ?LOGT("test broker get subscribed topics=~p~n", [Response]),
+    {reply, Response, State};
+
+handle_call({unsubscribe, Topic}, _From, State=#state{subscriber = SubList}) ->
+    ?LOGT("test broker unsubscribe Topic=~p~n", [Topic]),
+    is_binary(Topic) orelse error("Topic should be a binary"),
+    NewSubList = proplists:delete(Topic, SubList),
+    {reply, {ok, []}, State#state{subscriber = NewSubList}};
+
+
+handle_call({publish, {Topic, Msg, MatchedTopicFilter}}, _From, State=#state{subscriber = SubList}) ->
+    (is_binary(Topic) and is_binary(Msg)) orelse error("Topic and Msg should be binary"),
+    Pid = proplists:get_value(MatchedTopicFilter, SubList),
+    ?LOGT("test broker publish topic=~p, Msg=~p, Pid=~p, MatchedTopicFilter=~p, SubList=~p~n", [Topic, Msg, Pid, MatchedTopicFilter, SubList]),
+    (Pid == undefined) andalso ?LOGT("!!!!! this topic ~p has never been subscribed, please specify a valid topic filter", [MatchedTopicFilter]),
+    ?assertNotEqual(undefined, Pid),
+    Pid ! {deliver, #message{topic = Topic, payload = Msg}},
+    {reply, ok, State};
+
+handle_call(stop, _From, State) ->
+    {stop, normal, stopped, State};
+
+handle_call(Req, _From, State) ->
+    ?LOGT("test_broker_server: ignore call Req=~p~n", [Req]),
+    {reply, {error, badreq}, State}.
+
+
+handle_cast(Msg, State) ->
+    ?LOGT("test_broker_server: ignore cast msg=~p~n", [Msg]),
+    {noreply, State}.
+
+handle_info(Info, State) ->
+    ?LOGT("test_broker_server: ignore info=~p~n", [Info]),
+    {noreply, State}.
+
+terminate(Reason, _State) ->
+    ?LOGT("test_broker_server: terminate Reason=~p~n", [Reason]),
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+
+
+
+subscribed_topics([], Acc) ->
+    Acc;
+subscribed_topics([{Topic,_Pid}|T], Acc) ->
+    subscribed_topics(T, [Topic|Acc]).
+
+
+
+
+-record(keepalive, {statfun, statval, tsec, tmsg, tref, repeat = 0}).
+
+-type(keepalive() :: #keepalive{}).
+
+%% @doc Start a keepalive
+-spec(start(fun(), integer(), any()) -> undefined | keepalive()).
+start(_, 0, _) ->
+    undefined;
+start(StatFun, TimeoutSec, TimeoutMsg) ->
+    {ok, StatVal} = StatFun(),
+    #keepalive{statfun = StatFun, statval = StatVal,
+        tsec = TimeoutSec, tmsg = TimeoutMsg,
+        tref = timer(TimeoutSec, TimeoutMsg)}.
+
+%% @doc Check keepalive, called when timeout.
+-spec(check(keepalive()) -> {ok, keepalive()} | {error, any()}).
+check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repeat}) ->
+    case StatFun() of
+        {ok, NewVal} ->
+            if NewVal =/= LastVal ->
+                {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = 0})};
+                Repeat < 1 ->
+                    {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = Repeat + 1})};
+                true ->
+                    {error, timeout}
+            end;
+        {error, Error} ->
+            {error, Error}
+    end.
+
+resume(KeepAlive = #keepalive{tsec = TimeoutSec, tmsg = TimeoutMsg}) ->
+    KeepAlive#keepalive{tref = timer(TimeoutSec, TimeoutMsg)}.
+
+%% @doc Cancel Keepalive
+-spec(cancel(keepalive()) -> ok).
+cancel(#keepalive{tref = TRef}) ->
+    cancel(TRef);
+cancel(undefined) ->
+    ok;
+cancel(TRef) ->
+        catch erlang:cancel_timer(TRef).
+
+timer(Sec, Msg) ->
+    erlang:send_after(timer:seconds(Sec), self(), Msg).
+
+
+log(Format, Args) ->
+    logger:debug(Format, Args).
diff --git a/rebar.config.erl b/rebar.config.erl
index 4098bb7ab..ac84ead5a 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -346,6 +346,7 @@ relx_overlay(ReleaseType) ->
     , {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup
     , {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup
     , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup
+    , {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}
     , {template, "bin/emqx.cmd", "bin/emqx.cmd"}
     , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
     , {copy, "bin/nodetool", "bin/nodetool"}

From 7e00805af3767bf38d2088239b83b58ccddb0a99 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Tue, 3 Aug 2021 14:30:31 +0800
Subject: [PATCH 27/27] chore(gw-lwm2m): remove needless files

---
 .../emqx_gateway/src/lwm2m/emqx_lwm2m.app.src |   7 --
 .../emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl |  43 -------
 .../src/lwm2m/emqx_lwm2m_cm_sup.erl           |  41 ------
 .../src/lwm2m/emqx_lwm2m_coap_server.erl      | 117 ------------------
 .../src/lwm2m/emqx_lwm2m_protocol.erl         |   2 +-
 .../emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl |  39 ------
 6 files changed, 1 insertion(+), 248 deletions(-)
 delete mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src
 delete mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
 delete mode 100644 apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl

diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src
deleted file mode 100644
index 209499fc5..000000000
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src
+++ /dev/null
@@ -1,7 +0,0 @@
-{application,emqx_lwm2m,
-             [{description,"EMQ X LwM2M Gateway"},
-              {vsn, "4.3.2"}, % strict semver, bump manually!
-              {modules,[]},
-              {registered,[emqx_lwm2m_sup]},
-              {applications,[kernel,stdlib,lwm2m_coap]},
-              {mod,{emqx_lwm2m_app,[]}}]}.
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl
deleted file mode 100644
index 46f29c208..000000000
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_app.erl
+++ /dev/null
@@ -1,43 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_lwm2m_app).
-
--behaviour(application).
-
--emqx_plugin(protocol).
-
--export([ start/2
-        , stop/1
-        , prep_stop/1
-        ]).
-
--include("src/lwm2m/include/emqx_lwm2m.hrl").
-
-start(_Type, _Args) ->
-    Pid = emqx_lwm2m_sup:start_link(),
-    _ = lwm2m_coap_server:start_registry(),
-    lwm2m_coap_server_registry:add_handler([<<"rd">>], emqx_lwm2m_coap_resource, undefined),
-    emqx_lwm2m_coap_server:start(application:get_all_env(?APP)),
-    Pid.
-
-prep_stop(State) ->
-    lwm2m_coap_server_registry:remove_handler([<<"rd">>], emqx_lwm2m_coap_resource, undefined),
-    emqx_lwm2m_coap_server:stop(application:get_all_env(?APP)),
-    State.
-
-stop(_State) ->
-    ok.
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl
deleted file mode 100644
index b55f4f33c..000000000
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm_sup.erl
+++ /dev/null
@@ -1,41 +0,0 @@
-
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_lwm2m_cm_sup).
-
--behaviour(supervisor).
-
--export([start_link/0]).
-
--export([init/1]).
-
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
-init([]) ->
-    CM = #{id => emqx_lwm2m_cm,
-           start => {emqx_lwm2m_cm, start_link, []},
-           restart => permanent,
-           shutdown => 5000,
-           type => worker,
-           modules => [emqx_lwm2m_cm]},
-    SupFlags = #{strategy => one_for_one,
-                 intensity => 100,
-                 period => 10
-                },
-    {ok, {SupFlags, [CM]}}.
-
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
deleted file mode 100644
index b2f793364..000000000
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_server.erl
+++ /dev/null
@@ -1,117 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_lwm2m_coap_server).
-
--include("src/lwm2m/include/emqx_lwm2m.hrl").
-
--export([ start/1
-        , stop/1
-        ]).
-
--export([ start_listener/1
-        , start_listener/3
-        , stop_listener/1
-        , stop_listener/2
-        ]).
-
--include_lib("emqx/include/logger.hrl").
-
-start(Envs) ->
-    start_listeners(Envs).
-
-stop(Envs) ->
-    stop_listeners(Envs).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listeners(Envs) ->
-    lists:foreach(fun start_listener/1, listeners_confs(Envs)).
-
-stop_listeners(Envs) ->
-    lists:foreach(fun stop_listener/1, listeners_confs(Envs)).
-
-start_listener({Proto, ListenOn, Opts}) ->
-    case start_listener(Proto, ListenOn, Opts) of
-        {ok, _Pid} ->
-            ?ULOG("Start lwm2m:~s listener on ~s successfully.~n",
-                  [Proto, format(ListenOn)]);
-        {error, Reason} ->
-            ?ELOG("Failed to start lwm2m:~s listener on ~s: ~0p~n",
-                  [Proto, format(ListenOn), Reason]),
-            error(Reason)
-    end.
-
-start_listener(udp, ListenOn, Opts) ->
-    lwm2m_coap_server:start_udp('lwm2m:udp', ListenOn, Opts);
-start_listener(dtls, ListenOn, Opts) ->
-    lwm2m_coap_server:start_dtls('lwm2m:dtls', ListenOn, Opts).
-
-stop_listener({Proto, ListenOn, _Opts}) ->
-    Ret = stop_listener(Proto, ListenOn),
-    case Ret of
-        ok -> ?ULOG("Stop lwm2m:~s listener on ~s successfully.~n",
-                    [Proto, format(ListenOn)]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop lwm2m:~s listener on ~s: ~0p~n",
-                  [Proto, format(ListenOn), Reason])
-    end,
-    Ret.
-
-stop_listener(udp, ListenOn) ->
-    lwm2m_coap_server:stop_udp('lwm2m:udp', ListenOn);
-stop_listener(dtls, ListenOn) ->
-    lwm2m_coap_server:stop_dtls('lwm2m:dtls', ListenOn).
-
-listeners_confs(Envs) ->
-    listeners_confs(udp, Envs) ++ listeners_confs(dtls, Envs).
-
-listeners_confs(udp, Envs) ->
-    Udps = proplists:get_value(bind_udp, Envs, []),
-    Opts = proplists:get_value(options, Envs, []),
-    [{udp, Port, [{udp_options, InetOpts ++ Opts}] ++ get_lwm2m_opts(Envs)} || {Port, InetOpts} <- Udps];
-
-listeners_confs(dtls, Envs) ->
-    Dtls = proplists:get_value(bind_dtls, Envs, []),
-    Opts = proplists:get_value(dtls_opts, Envs, []),
-    [{dtls, Port, [{dtls_options, InetOpts ++ Opts}] ++ get_lwm2m_opts(Envs)} || {Port, InetOpts} <- Dtls].
-
-format(Port) when is_integer(Port) ->
-    io_lib:format("0.0.0.0:~w", [Port]);
-format({Addr, Port}) when is_list(Addr) ->
-    io_lib:format("~s:~w", [Addr, Port]);
-format({Addr, Port}) when is_tuple(Addr) ->
-    io_lib:format("~s:~w", [inet:ntoa(Addr), Port]).
-
-get_lwm2m_opts(Envs) ->
-    LifetimeMax = proplists:get_value(lifetime_max, Envs, 315360000),
-    LifetimeMin = proplists:get_value(lifetime_min, Envs, 0),
-    Mountpoint = proplists:get_value(mountpoint, Envs, ""),
-    Sockport = proplists:get_value(port, Envs, 5683),
-    AutoObserve = proplists:get_value(auto_observe, Envs, []),
-    QmodeTimeWindow = proplists:get_value(qmode_time_window, Envs, []),
-    Topics = proplists:get_value(topics, Envs, []),
-    PublishCondition = proplists:get_value(update_msg_publish_condition, Envs, contains_object_list),
-    [{lifetime_max, LifetimeMax},
-     {lifetime_min, LifetimeMin},
-     {mountpoint, list_to_binary(Mountpoint)},
-     {port, Sockport},
-     {auto_observe, AutoObserve},
-     {qmode_time_window, QmodeTimeWindow},
-     {update_msg_publish_condition, PublishCondition},
-     {topics, Topics}].
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
index 2ee61ec70..f14d6cb73 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
@@ -301,7 +301,7 @@ auto_observe_object_list(Expected, Registered) ->
 send_auto_observe(CoapPid, RegInfo, EndpointName) ->
     %% - auto observe the objects
     Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), 
-    case proplists:get_value(auto_observe, Envs, false) of
+    case maps:get(auto_observe, Envs, false) of
         false ->
             ?LOG(info, "Auto Observe Disabled", []);
         TrueOrObjList ->
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl
deleted file mode 100644
index 0546e0080..000000000
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_sup.erl
+++ /dev/null
@@ -1,39 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_lwm2m_sup).
-
--behaviour(supervisor).
-
--export([ start_link/0
-        , init/1
-        ]).
-
--define(CHILD(M), {M, {M, start_link, []}, permanent, 5000, worker, [M]}).
-
-
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
-init(_Args) ->
-    CmSup = #{id => emqx_lwm2m_cm_sup,
-              start => {emqx_lwm2m_cm_sup, start_link, []},
-              restart => permanent,
-              shutdown => infinity,
-              type => supervisor,
-              modules => [emqx_lwm2m_cm_sup]
-            },
-    {ok, { {one_for_all, 10, 3600}, [?CHILD(emqx_lwm2m_xml_object_db), CmSup] }}.