Merge branch 'master' into EMQX-871-872

.github/workflows/build_packages.yaml

@@ -216,6 +216,8 @@ jobs:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        erl_otp:
+          - 23.2.7.2-emqx-2
         arch:
           - amd64
           - arm64
@@ -295,7 +297,7 @@ jobs:
         done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        ERL_OTP: erl${{ matrix.erl_otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -336,17 +338,14 @@ jobs:
 
   docker:
     runs-on: ubuntu-20.04
-
     needs: prepare
 
     strategy:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
+        erl_otp:
-          - [amd64, x86_64]
+          - 23.2.7.2-emqx-2
-          - [arm64v8, aarch64]
-          - [arm32v7, arm]
 
     steps:
     - uses: actions/download-artifact@v2
@@ -355,22 +354,49 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
+    - uses: docker/setup-buildx-action@v1
-      env:
+    - uses: docker/setup-qemu-action@v1
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
-        name: ${{ matrix.profile }}
+        image: tonistiigi/binfmt:latest
-        path: source/_packages/${{ matrix.profile }}/.
+        platforms: all
+    - name: build emqx docker image
+      if: github.event_name != 'release'
+      env:
+        ERL_OTP: erl${{ matrix.erl_otp }}
+        PROFILE: ${{ matrix.profile }}
+      run: |
+        cd source
+        PKG_VSN="$(./pkg-vsn.sh)"
+        docker buildx build --no-cache \
+           --platform=linux/amd64,linux/arm64 \
+           --build-arg PKG_VSN=$PKG_VSN  \
+           --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine  \
+           --build-arg RUN_FROM=alpine:3.14 \
+           --build-arg EMQX_NAME=$PROFILE \
+           --tag emqx/$PROFILE:$PKG_VSN \
+           -f deploy/docker/Dockerfile .
+    - uses: docker/login-action@v1
+      if: github.event_name == 'release'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - name: build emqx docker image
+      if: github.event_name == 'release'
+      env:
+        ERL_OTP: erl${{ matrix.erl_otp }}
+        PROFILE: ${{ matrix.profile }}
+      run: |
+        cd source
+        PKG_VSN="$(./pkg-vsn.sh)"
+        docker buildx build --no-cache \
+           --platform=linux/amd64,linux/arm64 \
+           --build-arg PKG_VSN=$PKG_VSN  \
+           --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-alpine  \
+           --build-arg RUN_FROM=alpine:3.14 \
+           --build-arg EMQX_NAME=$PROFILE \
+           --tag emqx/$PROFILE:$PKG_VSN \
+           -f deploy/docker/Dockerfile \
+           --push .
 
   delete-artifact:
     runs-on: ubuntu-20.04

.github/workflows/run_fvt_tests.yaml

@@ -24,14 +24,16 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
+              echo "PROFILE=emqx" >> $GITHUB_ENV
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make $PROFILE-docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -79,11 +81,13 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "PROFILE=emqx" >> $GITHUB_ENV
             fi
         - name: make emqx image
-          run: make docker
+          run: make $PROFILE-docker
         - name: install k3s
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"

Makefile

@@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
+export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.3
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 endif
@@ -153,7 +153,14 @@ run: $(PROFILE) quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
 
-include docker.mk
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
 
 conf-segs:
 	@scripts/merge-config.escript

apps/emqx/include/logger.hrl

@@ -14,10 +14,10 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
+-ifndef(EMQX_LOGGER_HRL).
+-define(EMQX_LOGGER_HRL, true).
+
 %% debug | info | notice | warning | error | critical | alert | emergency
-
--compile({parse_transform, emqx_logger}).
-
 -define(DEBUG(Format), ?LOG(debug, Format, [])).
 -define(DEBUG(Format, Args), ?LOG(debug, Format, Args)).
 
@@ -41,10 +41,28 @@
 
 -define(LOG(Level, Format), ?LOG(Level, Format, [])).
 
--define(LOG(Level, Format, Args),
+-define(LOG(Level, Format, Args, Meta),
-        begin
+        %% check 'allow' here so we do not have to pass an anonymous function
-          (logger:log(Level,#{},#{report_cb => fun(_) -> {'$logger_header'()++(Format), (Args)} end,
+        %% down to logger which may cause `badfun` exception during upgrade
-                                  mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
+        case logger:allow(Level, ?MODULE) of
-                                  line => ?LINE}))
+            true ->
+                logger:log(Level, (Format), (Args),
+                           (Meta)#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+                                  , line => ?LINE
+                                  });
+            false ->
+                ok
         end).
 
+-define(LOG(Level, Format, Args), ?LOG(Level, Format, Args, #{})).
+
+%% structured logging
+-define(SLOG(Level, Data),
+        logger:log(Level, Data, #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+                                 , line => ?LINE})).
+
+%% print to 'user' group leader
+-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
+-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).
+
+-endif.

apps/emqx/rebar.config

@@ -15,7 +15,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.5"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}

apps/emqx/src/emqx.erl

@@ -20,7 +20,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[EMQ X]").
 
 %% Start/Stop the application
 -export([ start/0
@@ -77,8 +76,8 @@ set_debug_secret(PathToSecretFile) ->
                 catch _ : _ -> error({badfile, PathToSecretFile})
                 end;
             {error, Reason} ->
-                io:format("Failed to read debug_info encryption key file ~s: ~p~n",
+                ?ULOG("Failed to read debug_info encryption key file ~s: ~p~n",
-                          [PathToSecretFile, Reason]),
+                      [PathToSecretFile, Reason]),
                 error(Reason)
         end,
     F = fun(init) -> ok;

apps/emqx/src/emqx_alarm.erl

@@ -22,8 +22,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Alarm Handler]").
-
 %% Mnesia bootstrap
 -export([mnesia/1]).
 

apps/emqx/src/emqx_alarm_handler.erl

@@ -21,7 +21,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Alarm Handler]").
 
 %% gen_event callbacks
 -export([ init/1

apps/emqx/src/emqx_app.erl

@@ -27,6 +27,8 @@
         ]).
 
 -include("emqx.hrl").
+-include("emqx_release.hrl").
+-include("logger.hrl").
 
 -define(APP, emqx).
 
@@ -37,7 +39,6 @@
                      , ?MOD_DELAYED_SHARD
                      ]).
 
--include("emqx_release.hrl").
 
 %%--------------------------------------------------------------------
 %% Application callbacks
@@ -113,17 +114,17 @@ start_ce_modules() ->
 print_otp_version_warning() -> ok.
 -else.
 print_otp_version_warning() ->
-    io:format("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n",
+    ?ULOG("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n",
-              [?OTP_RELEASE]).
+          [?OTP_RELEASE]).
 -endif. % OTP_RELEASE
 
 -ifndef(TEST).
 
 print_banner() ->
-    io:format("Starting ~s on node ~s~n", [?APP, node()]).
+    ?ULOG("Starting ~s on node ~s~n", [?APP, node()]).
 
 print_vsn() ->
-    io:format("~s ~s is running now!~n", [get_description(), get_release()]).
+    ?ULOG("~s ~s is running now!~n", [get_description(), get_release()]).
 
 -else. % TEST
 

apps/emqx/src/emqx_banned.erl

@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Banned]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

apps/emqx/src/emqx_broker.erl

@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include("emqx_mqtt.hrl").
 
--logger_header("[Broker]").
 
 -export([start_link/2]).
 

apps/emqx/src/emqx_broker_helper.erl

@@ -21,7 +21,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Broker Helper]").
 
 -export([start_link/0]).
 

apps/emqx/src/emqx_channel.erl

@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Channel]").
 
 -ifdef(TEST).
 -compile(export_all).

apps/emqx/src/emqx_cm.erl

@@ -24,7 +24,6 @@
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--logger_header("[CM]").
 
 -export([start_link/0]).
 

apps/emqx/src/emqx_cm_registry.erl

@@ -23,7 +23,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Registry]").
 
 -export([start_link/0]).
 

apps/emqx/src/emqx_config.erl

@@ -209,7 +209,7 @@ init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) ->
 check_config(SchemaModule, RawConf) ->
     Opts = #{return_plain => true,
              nullable => true,
-             is_richmap => false
+             format => map
             },
     {AppEnvs, CheckedConf} =
         hocon_schema:map_translate(SchemaModule, RawConf, Opts),

apps/emqx/src/emqx_connection.erl

@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--logger_header("[MQTT]").
 
 -ifdef(TEST).
 -compile(export_all).

apps/emqx/src/emqx_ctl.erl

@@ -21,7 +21,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[Ctl]").
 
 -export([start_link/0, stop/0]).
 

apps/emqx/src/emqx_flapping.erl

@@ -22,8 +22,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[Flapping]").
-
 -export([start_link/0, stop/0]).
 
 %% API
@@ -162,4 +160,4 @@ start_timer(Zone) ->
 start_timers() ->
     lists:foreach(fun({Zone, _ZoneConf}) ->
             start_timer(Zone)
-        end, maps:to_list(emqx_config:get([zones], #{}))).
+        end, maps:to_list(emqx_config:get([zones], #{}))).

apps/emqx/src/emqx_hooks.erl

@@ -22,7 +22,6 @@
 -include("types.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--logger_header("[Hooks]").
 
 -export([ start_link/0
         , stop/0

apps/emqx/src/emqx_listeners.erl

@@ -18,6 +18,7 @@
 -module(emqx_listeners).
 
 -include("emqx_mqtt.hrl").
+-include("logger.hrl").
 
 %% APIs
 -export([ list/0
@@ -108,14 +109,13 @@ start_listener(ZoneName, ListenerName, #{type := Type, bind := Bind} = Conf) ->
         {error, {already_started, Pid}} ->
             {error, {already_started, Pid}};
         {error, Reason} ->
-            io:format(standard_error, "Failed to start ~s listener ~s on ~s: ~0p~n",
+            ?ELOG("Failed to start ~s listener ~s on ~s: ~0p~n",
-                      [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]),
+                  [Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]),
             error(Reason)
     end.
 
 -ifndef(TEST).
-console_print(Fmt, Args) ->
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-    io:format(Fmt, Args).
 -else.
 console_print(_Fmt, _Args) -> ok.
 -endif.

apps/emqx/src/emqx_logger.erl

@@ -56,8 +56,6 @@
         , stop_log_handler/1
         ]).
 
--export([parse_transform/2]).
-
 -type(peername_str() :: list()).
 -type(logger_dst() :: file:filename() | console | unknown).
 -type(logger_handler_info() :: #{
@@ -234,13 +232,6 @@ set_log_level(Level) ->
         {error, Error} -> {error, {primary_logger_level, Error}}
     end.
 
-%% @doc The parse transform for prefixing a module-specific logger header to the logs.
-%% The logger header can be specified by "-logger_header(Header)", where Header
-%% must be a string (list).
-%% @end
-parse_transform(AST, _Opts) ->
-    trans(AST, "", []).
-
 %%--------------------------------------------------------------------
 %% Internal Functions
 %%--------------------------------------------------------------------
@@ -307,31 +298,3 @@ list_stopped_handler_config() ->
         undefined -> [];
         ConfList -> maps:values(ConfList)
     end.
-
-%% @doc The following parse-transforms stripped off the module attribute named
-%% `-logger_header(Header)` (if there's one) from the source code, and then
-%% generate a function named '$logger_header'/0, which returns the logger header.
-%% @end
-trans([], LogHeader, ResAST) ->
-    lists:reverse([header_fun(LogHeader) | ResAST]);
-trans([{eof, L} | AST], LogHeader, ResAST) ->
-    lists:reverse([{eof, L}, header_fun(LogHeader) | ResAST]) ++ AST;
-trans([{attribute, _, module, _Mod} = M | AST], Header, ResAST) ->
-    trans(AST, Header, [export_header_fun(), M | ResAST]);
-trans([{attribute, _, logger_header, Header} | AST], _, ResAST) ->
-    io_lib:printable_list(Header) orelse erlang:error({invalid_string, Header}),
-    trans(AST, Header, ResAST);
-trans([F | AST], LogHeader, ResAST) ->
-    trans(AST, LogHeader, [F | ResAST]).
-
-export_header_fun() ->
-    {attribute,erl_anno:new(0),export,[{'$logger_header',0}]}.
-
-header_fun(LogHeader) ->
-    L = erl_anno:new(0),
-    {function,L,'$logger_header',0,
-         [{clause,L,
-             [], [], [{string,L,pad(LogHeader)}]}]}.
-
-pad("") -> "";
-pad(Str) -> Str ++ " ".

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -42,6 +42,9 @@
 
 -elvis([{elvis_style, no_nested_try_catch, #{ ignore => [emqx_logger_jsonfmt]}}]).
 
+%% this is what used when calling logger:log(Level, Report, Meta).
+-define(DEFAULT_FORMATTER, fun logger:format_otp_report/1).
+
 -type config() :: #{depth       => pos_integer() | unlimited,
                     report_cb   => logger:report_cb(),
                     single_line => boolean()}.
@@ -55,7 +58,11 @@ format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0
 
 format(Msg, Meta, Config) ->
     Data0 =
-        try Meta#{msg => format_msg(Msg, Meta, Config)}
+        try maybe_format_msg(Msg, Meta, Config) of
+            Map when is_map(Map) ->
+                maps:merge(Map, Meta);
+            Bin when is_binary(Bin) ->
+                Meta#{msg => Bin}
         catch
             C:R:S ->
                 Meta#{ msg => "emqx_logger_jsonfmt_format_error"
@@ -68,12 +75,26 @@ format(Msg, Meta, Config) ->
     Data = maps:without([report_cb], Data0),
     jiffy:encode(json_obj(Data, Config)).
 
+maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
+    case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
+        true ->
+            %% reporting a map without a customised format function
+            Report;
+        false ->
+            format_msg(Msg, Meta, Config)
+    end;
+maybe_format_msg(Msg, Meta, Config) ->
+    format_msg(Msg, Meta, Config).
+
 format_msg({string, Chardata}, Meta, Config) ->
+    %% already formatted
     format_msg({"~ts", [Chardata]}, Meta, Config);
 format_msg({report, _} = Msg, Meta, #{report_cb := Fun} = Config)
   when is_function(Fun,1); is_function(Fun,2) ->
+    %% a format callback function in config, no idea when this happens, but leaving it
     format_msg(Msg, Meta#{report_cb => Fun}, maps:remove(report_cb, Config));
 format_msg({report, Report}, #{report_cb := Fun} = Meta, Config) when is_function(Fun, 1) ->
+    %% a format callback function of arity 1
     case Fun(Report) of
         {Format, Args} when is_list(Format), is_list(Args) ->
             format_msg({Format, Args}, maps:remove(report_cb, Meta), Config);
@@ -84,6 +105,7 @@ format_msg({report, Report}, #{report_cb := Fun} = Meta, Config) when is_functio
              }
     end;
 format_msg({report, Report}, #{report_cb := Fun}, Config) when is_function(Fun, 2) ->
+    %% a format callback function of arity 2
     case Fun(Report, maps:with([depth, single_line], Config)) of
         Chardata when ?IS_STRING(Chardata) ->
             try
@@ -197,12 +219,18 @@ json_obj(Data, Config) ->
                       json_kv(K, V, D, Config)
               end, maps:new(), Data).
 
-json_kv(mfa, {M, F, A}, Data, _Config) -> %% emqx/snabbkaffe
+json_kv(mfa, {M, F, A}, Data, _Config) ->
     maps:put(mfa, <<(atom_to_binary(M, utf8))/binary, $:,
                     (atom_to_binary(F, utf8))/binary, $/,
                     (integer_to_binary(A))/binary>>, Data);
 json_kv('$kind', Kind, Data, Config) -> %% snabbkaffe
     maps:put(msg, json(Kind, Config), Data);
+json_kv(gl, _, Data, _Config) ->
+    %% drop gl because it's not interesting
+    Data;
+json_kv(file, _, Data, _Config) ->
+    %% drop 'file' because we have mfa
+    Data;
 json_kv(K0, V, Data, Config) ->
     K = json_key(K0),
     case is_map(V) of

apps/emqx/src/emqx_logger_textfmt.erl

@@ -19,25 +19,24 @@
 -export([format/2]).
 -export([check_config/1]).
 
-%% metadata fields which we do not wish to merge into log data
--define(WITHOUT_MERGE,
-        [ report_cb % just a callback
-        , time      % formatted as a part of templated message
-        , peername  % formatted as a part of templated message
-        , clientid  % formatted as a part of templated message
-        , gl        % not interesting
-        ]).
-
 check_config(X) -> logger_formatter:check_config(X).
 
-format(#{msg := Msg0, meta := Meta} = Event, Config) ->
+format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) ->
-    Msg = maybe_merge(Msg0, Meta),
+    logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config);
-    logger_formatter:format(Event#{msg := Msg}, Config).
+format(#{msg := {Fmt, Args}, meta := Meta} = Event, Config) when is_list(Fmt) ->
+    {NewFmt, NewArgs} = enrich_fmt(Fmt, Args, Meta),
+    logger_formatter:format(Event#{msg := {NewFmt, NewArgs}}, Config).
 
-maybe_merge({report, Report}, Meta) when is_map(Report) ->
+enrich(Report, #{mfa := Mfa, line := Line}) ->
-    {report, maps:merge(Report, filter(Meta))};
+    Report#{mfa => mfa(Mfa), line => Line};
-maybe_merge(Report, _Meta) ->
+enrich(Report, _) -> Report.
-    Report.
 
-filter(Meta) ->
+enrich_fmt(Fmt, Args, #{mfa := Mfa, line := Line}) ->
-    maps:without(?WITHOUT_MERGE, Meta).
+    {Fmt ++ " mfa: ~s line: ~w", Args ++ [mfa(Mfa), Line]};
+enrich_fmt(Fmt, Args, _) ->
+    {Fmt, Args}.
+
+mfa({M, F, A}) ->
+    <<(atom_to_binary(M, utf8))/binary, $:,
+      (atom_to_binary(F, utf8))/binary, $/,
+      (integer_to_binary(A))/binary>>.

apps/emqx/src/emqx_metrics.erl

@@ -24,7 +24,6 @@
 -include("emqx_mqtt.hrl").
 -include("emqx.hrl").
 
--logger_header("[Metrics]").
 
 -export([ start_link/0
         , stop/0

apps/emqx/src/emqx_os_mon.erl

@@ -20,7 +20,6 @@
 
 -include("logger.hrl").
 
--logger_header("[OS_MON]").
 
 -export([start_link/0]).
 

apps/emqx/src/emqx_plugins.erl

@@ -19,7 +19,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Plugins]").
 
 -export([ load/0
         , load/1

apps/emqx/src/emqx_pool.erl

@@ -21,7 +21,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Pool]").
 
 %% APIs
 -export([start_link/2]).

apps/emqx/src/emqx_psk.erl

@@ -18,7 +18,6 @@
 
 -include("logger.hrl").
 
--logger_header("[PSK]").
 
 %% SSL PSK Callbacks
 -export([lookup/3]).

apps/emqx/src/emqx_router.erl

@@ -23,7 +23,6 @@
 -include("types.hrl").
 -include_lib("ekka/include/ekka.hrl").
 
--logger_header("[Router]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

apps/emqx/src/emqx_router_helper.erl

@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Router Helper]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

apps/emqx/src/emqx_rule_actions_trans.erl

@@ -18,18 +18,14 @@ trans([Form | AST], ResAST) ->
   trans(AST, [Form | ResAST]).
 
 trans_func_clauses("on_action_create_" ++ _ = _FuncName , Clauses) ->
-  %io:format("~n[[transing function: ~p]]~n", [_FuncName]),
-  %io:format("~n-----old clauses:~n", []), merl:print(Clauses),
   NewClauses = [
     begin
       Bindings = lists:flatten(get_vars(Args) ++ get_vars(Body, lefth)),
       Body2 = append_to_result(Bindings, Body),
       {clause, LineNo, Args, Guards, Body2}
     end || {clause, LineNo, Args, Guards, Body} <- Clauses],
-  %io:format("~n-----new clauses: ~n"), merl:print(NewClauses),
   NewClauses;
 trans_func_clauses(_FuncName, Clauses) ->
-  %io:format("~n[[discarding function: ~p]]~n", [_FuncName]),
   Clauses.
 
 get_vars(Exprs) ->

apps/emqx/src/emqx_session.erl

@@ -48,7 +48,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Session]").
 
 -ifdef(TEST).
 -compile(export_all).

apps/emqx/src/emqx_shared_sub.erl

@@ -23,7 +23,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Shared Sub]").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

apps/emqx/src/emqx_stats.erl

@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[Stats]").
 
 %% APIs
 -export([ start_link/0

apps/emqx/src/emqx_sys.erl

@@ -22,7 +22,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[SYS]").
 
 -export([ start_link/0
         , stop/0

apps/emqx/src/emqx_sys_mon.erl

@@ -21,7 +21,6 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--logger_header("[SYSMON]").
 
 -export([start_link/0]).
 

apps/emqx/src/emqx_tracer.erl

@@ -19,7 +19,6 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
--logger_header("[Tracer]").
 
 %% APIs
 -export([ trace/2

apps/emqx/src/emqx_ws_connection.erl

@@ -22,7 +22,6 @@
 -include("logger.hrl").
 -include("types.hrl").
 
--logger_header("[MQTT/WS]").
 
 -ifdef(TEST).
 -compile(export_all).

apps/emqx/test/emqx_logger_SUITE.erl

@@ -25,19 +25,6 @@
 -define(a, "a").
 -define(SUPPORTED_LEVELS, [emergency, alert, critical, error, warning, notice, info, debug]).
 
--define(PARSE_TRANS_TEST_CODE,
-        "-module(mytest).\n"
-        "-logger_header(\"[MyTest]\").\n"
-        "-export([run/0]).\n"
-        "-compile({parse_transform, logger_header}).\n"
-        "run() -> '$logger_header'().").
-
--define(PARSE_TRANS_TEST_CODE2,
-        "-module(mytest).\n"
-        "-export([run/0]).\n"
-        "-compile({parse_transform, logger_header}).\n"
-        "run() -> '$logger_header'().").
-
 all() -> emqx_ct:all(?MODULE).
 
 init_per_testcase(_TestCase, Config) ->
@@ -132,37 +119,6 @@ t_start_stop_log_handler2(_) ->
     ?assertMatch({error, {not_started, default}},
                  ?LOGGER:stop_log_handler(default)).
 
-t_parse_transform(_) ->
-    {ok, Toks, _EndLine} = erl_scan:string(?PARSE_TRANS_TEST_CODE),
-    FormToks = split_toks_at_dot(Toks),
-    Forms = [case erl_parse:parse_form(Ts) of
-                 {ok, Form} ->
-                     Form;
-                 {error, Reason} ->
-                     erlang:error({parse_form_error, Ts, Reason})
-             end
-             || Ts <- FormToks],
-    %ct:log("=====: ~p", [Forms]),
-    AST1 = emqx_logger:parse_transform(Forms, []),
-    %ct:log("=====: ~p", [AST1]),
-    ?assertNotEqual(false, lists:keyfind('$logger_header', 3, AST1)).
-
-t_parse_transform_empty_header(_) ->
-    {ok, Toks, _EndLine} = erl_scan:string(?PARSE_TRANS_TEST_CODE2),
-    FormToks = split_toks_at_dot(Toks),
-    Forms = [case erl_parse:parse_form(Ts) of
-                 {ok, Form} ->
-                     Form;
-                 {error, Reason} ->
-                     erlang:error({parse_form_error, Ts, Reason})
-             end
-             || Ts <- FormToks],
-    %ct:log("=====: ~p", [Forms]),
-    AST2 = emqx_logger:parse_transform(Forms++[{eof, 15}], []),
-    %ct:log("=====: ~p", [AST2]),
-    ?assertNotEqual(false, lists:keyfind('$logger_header', 3, AST2)).
-
-
 t_set_metadata_peername(_) ->
     ?assertEqual(ok, ?LOGGER:set_metadata_peername("for_test")).
 

apps/emqx_authn/etc/emqx_authn.conf

@@ -1,4 +1,4 @@
-emqx_authn: {
+authentication: {
     enable: false
     authenticators: [
         # {

apps/emqx_authn/src/emqx_authn.erl

@@ -20,6 +20,7 @@
 
 -export([ enable/0
         , disable/0
+        , is_enabled/0
         ]).
 
 -export([authenticate/2]).
@@ -84,6 +85,14 @@ disable() ->
     emqx:unhook('client.authenticate', {?MODULE, authenticate, []}),
     ok.
 
+is_enabled() ->
+    Callbacks = emqx_hooks:lookup('client.authenticate'),
+    lists:any(fun({callback, {?MODULE, authenticate, []}, _, _}) ->
+                  true;
+                 (_) ->
+                  false
+              end, Callbacks).
+
 authenticate(Credential, _AuthResult) ->
     case mnesia:dirty_read(?CHAIN_TAB, ?CHAIN) of
         [#chain{authenticators = Authenticators}] ->

apps/emqx_authn/src/emqx_authn_api.erl

@@ -131,6 +131,27 @@ authentication_api() ->
                 },
                 <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>)
             }
+        },
+        get => #{
+            description => "Get status of authentication",
+            responses => #{
+                <<"200">> => #{
+                    description => <<"OK">>,
+                    content => #{
+                        'application/json' => #{
+                            schema => #{
+                                type => object,
+                                properties => #{
+                                    enabled => #{
+                                        type => boolean,
+                                        example => true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
         }
     },
     {"/authentication", Metadata, authentication}.
@@ -1153,17 +1174,20 @@ authentication(post, Request) ->
             serialize_error({invalid_parameter, enable});
         _ ->
             serialize_error({missing_parameter, enable})
-    end.
+    end;
+authentication(get, _Request) ->
+    Enabled = emqx_authn:is_enabled(),
+    {200, #{enabled => Enabled}}.
 
 authenticators(post, Request) ->
     {ok, Body, _} = cowboy_req:read_body(Request),
     AuthenticatorConfig = emqx_json:decode(Body, [return_maps]),
-    Config = #{<<"emqx_authn">> => #{
+    Config = #{<<"authentication">> => #{
                    <<"authenticators">> => [AuthenticatorConfig]
                }},
     NConfig = hocon_schema:check_plain(emqx_authn_schema, Config,
                                        #{nullable => true}),
-    #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
+    #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
     case emqx_authn:create_authenticator(?CHAIN, NAuthenticatorConfig) of
         {ok, Authenticator2} ->
             {201, Authenticator2};
@@ -1186,12 +1210,12 @@ authenticators2(put, Request) ->
     AuthenticatorID = cowboy_req:binding(id, Request),
     {ok, Body, _} = cowboy_req:read_body(Request),
     AuthenticatorConfig = emqx_json:decode(Body, [return_maps]),
-    Config = #{<<"emqx_authn">> => #{
+    Config = #{<<"authentication">> => #{
                    <<"authenticators">> => [AuthenticatorConfig]
                }},
     NConfig = hocon_schema:check_plain(emqx_authn_schema, Config,
                                        #{nullable => true}),
-    #{emqx_authn := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
+    #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig),
     case emqx_authn:update_or_create_authenticator(?CHAIN, AuthenticatorID, NAuthenticatorConfig) of
         {ok, Authenticator} ->
             {200, Authenticator};

apps/emqx_authn/src/emqx_authn_app.erl

@@ -36,8 +36,8 @@ stop(_State) ->
     ok.
 
 initialize() ->
-    AuthNConfig = emqx_config:get([emqx_authn], #{enable => false,
+    AuthNConfig = emqx_config:get([authentication], #{enable => false,
-                                                  authenticators => []}),
+                                                      authenticators => []}),
     initialize(AuthNConfig).
 
 initialize(#{enable := Enable, authenticators := AuthenticatorsConfig}) ->

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -28,9 +28,9 @@
 -export([ authenticator_name/1
         ]).
 
-structs() -> [ "emqx_authn" ].
+structs() -> [ "authentication" ].
 
-fields("emqx_authn") ->
+fields("authentication") ->
     [ {enable, fun enable/1}
     , {authenticators, fun authenticators/1}
     ].

apps/emqx_authn/test/emqx_authn_SUITE.erl

@@ -101,5 +101,7 @@ t_authenticate(_) ->
                    username => <<"myuser">>,
 			       password => <<"mypass">>},
     ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)),
+    ?assertEqual(false, emqx_authn:is_enabled()),
     emqx_authn:enable(),
+    ?assertEqual(true, emqx_authn:is_enabled()),
     ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo)).

apps/emqx_authz/src/emqx_authz.erl

@@ -20,7 +20,6 @@
 -include("emqx_authz.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[AuthZ]").
 
 -export([ register_metrics/0
         , init/0

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl

@@ -20,7 +20,6 @@
 -include("emqx_bridge_mqtt.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Bridge]").
 
 %% APIs
 -export([ start_link/0

apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl

@@ -116,7 +116,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
--logger_header("[Bridge]").
 
 %% same as default in-flight limit for emqtt
 -define(DEFAULT_BATCH_SIZE, 32).

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -27,6 +27,7 @@
 %% Authorization
 -export([authorize_appid/1]).
 
+-include_lib("emqx/include/logger.hrl").
 
 -define(BASE_PATH, "/api/v5").
 
@@ -67,7 +68,7 @@ start_listener({Proto, Port, Options}) ->
         dispatch => Dispatch},
     MinirestOptions = maps:merge(Minirest, RanchOptions),
     {ok, _} = minirest:start(listener_name(Proto), MinirestOptions),
-    io:format("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
+    ?ULOG("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]).
 
 apps() ->
     [App || {App, _, _} <- application:loaded_applications(),
@@ -90,7 +91,7 @@ ranch_opts(Port, Options0) ->
     maps:from_list([{port, Port} | Options]).
 
 stop_listener({Proto, Port, _}) ->
-    io:format("Stop dashboard listener on ~s successfully.~n",[format(Port)]),
+    ?ULOG("Stop dashboard listener on ~s successfully.~n", [format(Port)]),
     minirest:stop(listener_name(Proto)).
 
 listeners() ->

apps/emqx_exhook/.gitignore

@@ -0,0 +1,29 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
+rebar.lock
+data/
+*.conf.rendered
+*.pyc
+.DS_Store
+*.class
+Mnesia.nonode@nohost/
+src/emqx_exhook_pb.erl
+src/emqx_exhook_v_1_hook_provider_client.erl
+src/emqx_exhook_v_1_hook_provider_bhvr.erl

apps/emqx_exhook/docs/design-cn.md

@@ -0,0 +1,112 @@
+# 设计
+
+## 动机
+
+在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力：
+
+1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能
+2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能
+
+但在后续的支持中发现许多难以处理的问题：
+
+1. 有大量的编程语言需要支持，需要编写和维护如 Go, JavaScript, Lua.. 等语言的驱动。
+2. `erlport` 使用的操作系统的管道进行通信，这让用户代码只能部署在和 emqx 同一个操作系统上。部署方式受到了极大的限制。
+3. 用户程序的启动参数直接打包到 Broker 中，导致用户开发无法实时的进行调试，单步跟踪等。
+4. `erlport` 会占用 `stdin` `stdout`。
+
+因此，我们计划重构这部分的实现，其中主要的内容是：
+1. 使用 `gRPC` 替换 `erlport`。
+2. 将 `emqx-extension-hook` 重命名为 `emqx-exhook`
+
+
+旧版本的设计：[emqx-extension-hook design in v4.2.0](https://github.com/emqx/emqx-exhook/blob/v4.2.0/docs/design.md)
+
+## 设计
+
+架构如下：
+
+```
+  EMQ X                                    
++========================+                 +========+==========+
+|    ExHook              |                 |        |          |
+|   +----------------+   |      gRPC       | gRPC   |  User's  |
+|   |   gRPC Client  | ------------------> | Server |  Codes   |
+|   +----------------+   |    (HTTP/2)     |        |          |
+|                        |                 |        |          |
++========================+                 +========+==========+
+```
+
+`emqx-exhook` 通过 gRPC 的方式向用户部署的 gRPC 服务发送钩子的请求，并处理其返回的值。
+
+
+和 emqx 原生的钩子一致，emqx-exhook 也按照链式的方式执行：
+
+<img src="https://docs.emqx.net/broker/latest/cn/advanced/assets/chain_of_responsiblity.png" style="zoom:50%;" />
+
+### gRPC 服务示例
+
+用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中：
+
+```protobuff
+syntax = "proto3";
+
+package emqx.exhook.v1;
+
+service HookProvider {
+
+  rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {};
+
+  rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {};
+
+  rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {};
+
+  rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {};
+
+  rpc OnClientAuthorize(ClientAuthorizeRequest) returns (ValuedResponse) {};
+
+  rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {};
+
+  rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {};
+
+  rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {};
+
+  rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {};
+
+  rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {};
+
+  rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {};
+
+  rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {};
+}
+```
+
+### 配置文件示例
+
+```
+exhook: {
+    ## 配置 gRPC 服务地址 (HTTP)
+    ##
+    ## default 为服务器的名称
+    server.default: {
+        url: "http://127.0.0.1:9000"
+    }
+}
+```

apps/emqx_exhook/etc/emqx_exhook.conf

@@ -0,0 +1,14 @@
+##====================================================================
+## EMQ X Hooks
+##====================================================================
+
+exhook: {
+    server.default: {
+        url: "http://127.0.0.1:9000"
+        #ssl: {
+        #    cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem"
+        #    certfile: "{{ platform_etc_dir }}/certs/cert.pem"
+        #    keyfile: "{{ platform_etc_dir }}/certs/key.pem"
+        #}
+    }
+}

apps/emqx_exhook/priv/protos/exhook.proto

@@ -127,14 +127,14 @@ message ClientAuthorizeRequest {
 
   ClientInfo clientinfo = 1;
 
-  enum AuthzReqType {
+  enum AuthorizeReqType {
 
     PUBLISH = 0;
 
     SUBSCRIBE = 1;
   }
 
-  AuthzReqType type = 2;
+  AuthorizeReqType type = 2;
 
   string topic = 3;
 

apps/emqx_exhook/rebar.config

@@ -0,0 +1,41 @@
+%%-*- mode: erlang -*-
+{plugins,
+ [rebar3_proper,
+  {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}}
+]}.
+
+{deps,
+ [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}}
+]}.
+
+{grpc,
+ [{protos, ["priv/protos"]},
+  {gpb_opts, [{module_name_prefix, "emqx_"},
+              {module_name_suffix, "_pb"}]}
+]}.
+
+{provider_hooks,
+ [{pre, [{compile, {grpc, gen}},
+         {clean, {grpc, clean}}]}
+]}.
+
+{edoc_opts, [{preprocess, true}]}.
+
+{erl_opts, [warn_unused_vars,
+            warn_shadow_vars,
+            warn_unused_import,
+            warn_obsolete_guard,
+            debug_info,
+            {parse_transform}]}.
+
+{xref_checks, [undefined_function_calls, undefined_functions,
+               locals_not_used, deprecated_function_calls,
+               warnings_as_errors, deprecated_functions]}.
+{xref_ignores, [emqx_exhook_pb]}.
+
+{cover_enabled, true}.
+{cover_opts, [verbose]}.
+{cover_export_enabled, true}.
+{cover_excl_mods, [emqx_exhook_pb,
+                   emqx_exhook_v_1_hook_provider_bhvr,
+                   emqx_exhook_v_1_hook_provider_client]}.

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,6 +1,6 @@
 {application, emqx_exhook,
  [{description, "EMQ X Extension for Hook"},
-  {vsn, "4.3.2"},
+  {vsn, "5.0.0"},
   {modules, []},
   {registered, []},
   {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook.appup.src

@@ -0,0 +1,9 @@
+%% -*-: erlang -*-
+{VSN,
+ [
+    {<<".*">>, []}
+ ],
+ [
+    {<<".*">>, []}
+ ]
+}.

apps/emqx_exhook/src/emqx_exhook.erl

@@ -16,10 +16,9 @@
 
 -module(emqx_exhook).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook]").
 
 %% Mgmt APIs
 -export([ enable/2
@@ -41,13 +40,13 @@
 list() ->
     [server(Name) || Name <- running()].
 
--spec enable(atom()|string(), list()) -> ok | {error, term()}.
+-spec enable(atom()|string(), map()) -> ok | {error, term()}.
-enable(Name, Opts) ->
+enable(Name, Options) ->
     case lists:member(Name, running()) of
         true ->
             {error, already_started};
         _ ->
-            case emqx_exhook_server:load(Name, Opts) of
+            case emqx_exhook_server:load(Name, Options) of
                 {ok, ServiceState} ->
                     save(Name, ServiceState);
                 {error, Reason} ->
@@ -107,7 +106,6 @@ call_fold(Hookpoint, Req, AccFun, [ServiceName|More]) ->
 %%----------------------------------------------------------
 %% Storage
 
--compile({inline, [save/2]}).
 save(Name, ServiceState) ->
     Saved = persistent_term:get(?APP, []),
     persistent_term:put(?APP, lists:reverse([Name | Saved])),

apps/emqx_exhook/src/emqx_exhook_app.erl

@@ -18,9 +18,7 @@
 
 -behaviour(application).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
-
--emqx_plugin(extension).
 
 -define(CNTER, emqx_exhook_counter).
 
@@ -67,9 +65,10 @@ stop(_State) ->
 %%--------------------------------------------------------------------
 
 load_all_servers() ->
-    lists:foreach(fun({Name, Options}) ->
+    _ = maps:map(fun(Name, Options) ->
         load_server(Name, Options)
-    end, application:get_env(?APP, servers, [])).
+    end, emqx_config:get([exhook, server])),
+    ok.
 
 unload_all_servers() ->
     emqx_exhook:disable_all().

apps/emqx_exhook/src/emqx_exhook_cli.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_exhook_cli).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 
 -export([cli/1]).
 
@@ -30,7 +30,7 @@ cli(["server", "list"]) ->
 cli(["server", "enable", Name0]) ->
     if_enabled(fun() ->
         Name = list_to_atom(Name0),
-        case proplists:get_value(Name, application:get_env(?APP, servers, [])) of
+        case maps:get(Name, emqx_config:get([exhook, server]), undefined) of
             undefined ->
                 emqx_ctl:print("not_found~n");
             Opts ->

apps/emqx_exhook/src/emqx_exhook_handler.erl

@@ -16,11 +16,10 @@
 
 -module(emqx_exhook_handler).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook]").
 
 -export([ on_client_connect/2
         , on_client_connack/3
@@ -95,7 +94,7 @@ on_client_authenticate(ClientInfo, AuthResult) ->
     %%      The return value of `call_fold` just a bool, that has missed
     %%      detailed info too.
     %%
-    Bool = maps:get(auth_result, AuthResult, undefined) == success,
+    Bool = AuthResult == ok,
     Req = #{clientinfo => clientinfo(ClientInfo),
             result => Bool
            },
@@ -103,8 +102,8 @@ on_client_authenticate(ClientInfo, AuthResult) ->
     case call_fold('client.authenticate', Req,
                    fun merge_responsed_bool/2) of
         {StopOrOk, #{result := Result0}} when is_boolean(Result0) ->
-            Result = case Result0 of true -> success; _ -> not_authorized end,
+            Result = case Result0 of true -> ok; _ -> {error, not_authorized} end,
-            {StopOrOk, AuthResult#{auth_result => Result, anonymous => false}};
+            {StopOrOk, Result};
         _ ->
             {ok, AuthResult}
     end.

apps/emqx_exhook/src/emqx_exhook_schema.erl

@@ -0,0 +1,62 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_schema).
+
+-dialyzer(no_return).
+-dialyzer(no_match).
+-dialyzer(no_contracts).
+-dialyzer(no_unused).
+-dialyzer(no_fail_call).
+
+-include_lib("typerefl/include/types.hrl").
+
+-behaviour(hocon_schema).
+
+-export([structs/0, fields/1]).
+-export([t/1, t/3, t/4, ref/1]).
+
+structs() -> [server].
+
+fields(server) ->
+    [{"$name", t(ref(server_structs))}];
+
+fields(server_structs) ->
+    [ {url, t(string(), "emqx_exhook.url", "")}
+    , {ssl, t(ref(ssl_conf_group))}
+    ];
+
+fields(ssl_conf_group) ->
+    [ {cacertfile, string()}
+    , {certfile, string()}
+    , {keyfile, string()}
+    ].
+
+%% types
+
+t(Type) -> #{type => Type}.
+
+t(Type, Mapping, Default) ->
+    hoconsc:t(Type, #{mapping => Mapping, default => Default}).
+
+t(Type, Mapping, Default, OverrideEnv) ->
+    hoconsc:t(Type, #{ mapping => Mapping
+                     , default => Default
+                     , override_env => OverrideEnv
+                     }).
+
+ref(Field) ->
+    hoconsc:ref(?MODULE, Field).

apps/emqx_exhook/src/emqx_exhook_server.erl

@@ -16,10 +16,9 @@
 
 -module(emqx_exhook_server).
 
--include("src/exhook/include/emqx_exhook.hrl").
+-include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExHook Svr]").
 
 -define(CNTER, emqx_exhook_counter).
 -define(PB_CLIENT_MOD, emqx_exhook_v_1_hook_provider_client).
@@ -41,7 +40,7 @@
           %% Server name (equal to grpc client channel name)
           name :: server_name(),
           %% The server started options
-          options :: list(),
+          options :: options(),
           %% gRPC channel pid
           channel :: pid(),
           %% Registered hook names and options
@@ -75,13 +74,17 @@
 
 -export_type([server/0]).
 
+-type options() :: #{ url := uri_string:uri_string()
+                    , ssl => map()
+                    }.
+
 -dialyzer({nowarn_function, [inc_metrics/2]}).
 
 %%--------------------------------------------------------------------
 %% Load/Unload APIs
 %%--------------------------------------------------------------------
 
--spec load(atom(), list()) -> {ok, server()} | {error, term()} .
+-spec load(atom(), options()) -> {ok, server()} | {error, term()} .
 load(Name0, Opts0) ->
     Name = to_list(Name0),
     {SvrAddr, ClientOpts} = channel_opts(Opts0),
@@ -118,28 +121,33 @@ to_list(Name) when is_list(Name) ->
     Name.
 
 %% @private
-channel_opts(Opts) ->
+channel_opts(Opts = #{url := URL}) ->
-    Scheme = proplists:get_value(scheme, Opts),
+    case uri_string:parse(URL) of
-    Host = proplists:get_value(host, Opts),
+        #{scheme := <<"http">>, host := Host, port := Port} ->
-    Port = proplists:get_value(port, Opts),
+            {format_http_uri("http", Host, Port), #{}};
-    SvrAddr = format_http_uri(Scheme, Host, Port),
+        #{scheme := <<"https">>, host := Host, port := Port} ->
-    ClientOpts = case Scheme of
+            SslOpts =
-                     https ->
+                case maps:get(ssl, Opts, undefined) of
-                         SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])),
+                    undefined -> [];
-                         #{gun_opts =>
+                    MapOpts ->
-                           #{transport => ssl,
+                        filter(
-                             transport_opts => SslOpts}};
+                          [{cacertfile, maps:get(cacertfile, MapOpts, undefined)},
-                     _ -> #{}
+                           {certfile, maps:get(certfile, MapOpts, undefined)},
-                 end,
+                           {keyfile, maps:get(keyfile, MapOpts, undefined)}
-    {SvrAddr, ClientOpts}.
+                          ])
+                end,
+            {format_http_uri("https", Host, Port),
+             #{gun_opts => #{transport => ssl, transport_opts => SslOpts}}};
+        _ ->
+            error(bad_server_url)
+    end.
 
-format_http_uri(Scheme, Host0, Port) ->
+format_http_uri(Scheme, Host, Port) ->
-    Host = case is_tuple(Host0) of
-               true -> inet:ntoa(Host0);
-               _ -> Host0
-           end,
     lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])).
 
+filter(Ls) ->
+    [ E || E <- Ls, E /= undefined].
+
 -spec unload(server()) -> ok.
 unload(#server{name = Name, hookspec = HookSpecs}) ->
     _ = do_deinit(Name),

apps/emqx_exhook/test/emqx_exhook_SUITE.erl

@@ -0,0 +1,103 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(CONF_DEFAULT, <<"
+exhook: { server.default: { url: \"http://127.0.0.1:9000\" } }
+">>).
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+
+all() -> emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    _ = emqx_exhook_demo_svr:start(),
+    ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
+    emqx_ct_helpers:start_apps([emqx_exhook]),
+    Cfg.
+
+end_per_suite(_Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_exhook]),
+    emqx_exhook_demo_svr:stop().
+
+%%--------------------------------------------------------------------
+%% Test cases
+%%--------------------------------------------------------------------
+
+t_noserver_nohook(_) ->
+    emqx_exhook:disable(default),
+    ?assertEqual([], loaded_exhook_hookpoints()),
+    Opts = emqx_config:get([exhook, server, default]),
+    ok = emqx_exhook:enable(default, Opts),
+    ?assertNotEqual([], loaded_exhook_hookpoints()).
+
+t_cli_list(_) ->
+    meck_print(),
+    ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]]
+                , emqx_exhook_cli:cli(["server", "list"])
+                ),
+    unmeck_print().
+
+t_cli_enable_disable(_) ->
+    meck_print(),
+    ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
+    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
+    ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])),
+
+    ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
+    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
+    unmeck_print().
+
+t_cli_stats(_) ->
+    meck_print(),
+    _ = emqx_exhook_cli:cli(["server", "stats"]),
+    _ = emqx_exhook_cli:cli(x),
+    unmeck_print().
+
+%%--------------------------------------------------------------------
+%% Utils
+%%--------------------------------------------------------------------
+
+meck_print() ->
+    meck:new(emqx_ctl, [passthrough, no_history, no_link]),
+    meck:expect(emqx_ctl, print, fun(_) -> ok end),
+    meck:expect(emqx_ctl, print, fun(_, Args) -> Args end).
+
+unmeck_print() ->
+    meck:unload(emqx_ctl).
+
+loaded_exhook_hookpoints() ->
+    lists:filtermap(fun(E) ->
+        Name = element(2, E),
+        Callbacks = element(3, E),
+        case lists:any(fun is_exhook_callback/1, Callbacks) of
+            true -> {true, Name};
+            _ -> false
+        end
+    end, ets:tab2list(emqx_hooks)).
+
+is_exhook_callback(Cb) ->
+    Action = element(2, Cb),
+    emqx_exhook_handler == element(1, Action).

apps/emqx_exhook/test/emqx_exhook_demo_svr.erl

@@ -0,0 +1,339 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_demo_svr).
+
+-behavior(emqx_exhook_v_1_hook_provider_bhvr).
+
+%%
+-export([ start/0
+        , stop/0
+        , take/0
+        , in/1
+        ]).
+
+%% gRPC server HookProvider callbacks
+-export([ on_provider_loaded/2
+        , on_provider_unloaded/2
+        , on_client_connect/2
+        , on_client_connack/2
+        , on_client_connected/2
+        , on_client_disconnected/2
+        , on_client_authenticate/2
+        , on_client_authorize/2
+        , on_client_subscribe/2
+        , on_client_unsubscribe/2
+        , on_session_created/2
+        , on_session_subscribed/2
+        , on_session_unsubscribed/2
+        , on_session_resumed/2
+        , on_session_discarded/2
+        , on_session_takeovered/2
+        , on_session_terminated/2
+        , on_message_publish/2
+        , on_message_delivered/2
+        , on_message_dropped/2
+        , on_message_acked/2
+        ]).
+
+-define(PORT, 9000).
+-define(NAME, ?MODULE).
+
+%%--------------------------------------------------------------------
+%% Server APIs
+%%--------------------------------------------------------------------
+
+start() ->
+    Pid = spawn(fun mngr_main/0),
+    register(?MODULE, Pid),
+    {ok, Pid}.
+
+stop() ->
+    grpc:stop_server(?NAME),
+    ?MODULE ! stop.
+
+take() ->
+    ?MODULE ! {take, self()},
+    receive {value, V} -> V
+    after 5000 -> error(timeout) end.
+
+in({FunName, Req}) ->
+    ?MODULE ! {in, FunName, Req}.
+
+mngr_main() ->
+    application:ensure_all_started(grpc),
+    Services = #{protos => [emqx_exhook_pb],
+                 services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
+                },
+    Options = [],
+    Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
+    mngr_loop([Svr, queue:new(), queue:new()]).
+
+mngr_loop([Svr, Q, Takes]) ->
+    receive
+        {in, FunName, Req} ->
+            {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
+            mngr_loop([Svr, NQ1, NQ2]);
+        {take, From} ->
+            {NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
+            mngr_loop([Svr, NQ1, NQ2]);
+        stop ->
+            exit(normal)
+    end.
+
+reply(Q1, Q2) ->
+    case queue:len(Q1) =:= 0 orelse
+         queue:len(Q2) =:= 0 of
+        true -> {Q1, Q2};
+        _ ->
+            {{value, {Name, V}}, NQ1} = queue:out(Q1),
+            {{value, From}, NQ2} = queue:out(Q2),
+            From ! {value, {Name, V}},
+            {NQ1, NQ2}
+    end.
+
+%%--------------------------------------------------------------------
+%% callbacks
+%%--------------------------------------------------------------------
+
+-spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+
+on_provider_loaded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{hooks => [
+                     #{name => <<"client.connect">>},
+                     #{name => <<"client.connack">>},
+                     #{name => <<"client.connected">>},
+                     #{name => <<"client.disconnected">>},
+                     #{name => <<"client.authenticate">>},
+                     #{name => <<"client.authorize">>},
+                     #{name => <<"client.subscribe">>},
+                     #{name => <<"client.unsubscribe">>},
+                     #{name => <<"session.created">>},
+                     #{name => <<"session.subscribed">>},
+                     #{name => <<"session.unsubscribed">>},
+                     #{name => <<"session.resumed">>},
+                     #{name => <<"session.discarded">>},
+                     #{name => <<"session.takeovered">>},
+                     #{name => <<"session.terminated">>},
+                     #{name => <<"message.publish">>},
+                     #{name => <<"message.delivered">>},
+                     #{name => <<"message.acked">>},
+                     #{name => <<"message.dropped">>}]}, Md}.
+-spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_provider_unloaded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connect(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connack(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_connected(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_disconnected(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case Username of
+        <<"baduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, false}}, Md};
+        <<"gooduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, true}}, Md};
+        <<"normaluser">> ->
+            {ok, #{type => 'CONTINUE',
+                   value => {bool_result, true}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_client_authorize(emqx_exhook_pb:client_authorize_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_authorize(#{clientinfo := #{username := Username}} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case Username of
+        <<"baduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, false}}, Md};
+        <<"gooduser">> ->
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {bool_result, true}}, Md};
+        <<"normaluser">> ->
+            {ok, #{type => 'CONTINUE',
+                   value => {bool_result, true}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_subscribe(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_client_unsubscribe(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_created(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_subscribed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_unsubscribed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_resumed(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_discarded(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_takeovered(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_session_terminated(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_publish(#{message := #{from := From} = Msg} = Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    %% some cases for testing
+    case From of
+        <<"baduser">> ->
+            NMsg = Msg#{qos => 0,
+                        topic => <<"">>,
+                        payload => <<"">>
+                       },
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {message, NMsg}}, Md};
+        <<"gooduser">> ->
+            NMsg = Msg#{topic => From,
+                        payload => From},
+            {ok, #{type => 'STOP_AND_RETURN',
+                   value => {message, NMsg}}, Md};
+        _ ->
+            {ok, #{type => 'IGNORE'}, Md}
+    end.
+
+-spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_delivered(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_dropped(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.
+
+-spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata())
+    -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
+     | {error, grpc_cowboy_h:error_response()}.
+on_message_acked(Req, Md) ->
+    ?MODULE:in({?FUNCTION_NAME, Req}),
+    %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
+    {ok, #{}, Md}.

apps/emqx_exhook/test/props/prop_exhook_hooks.erl

@@ -0,0 +1,518 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(prop_exhook_hooks).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-import(emqx_ct_proper_types,
+        [ conninfo/0
+        , clientinfo/0
+        , sessioninfo/0
+        , message/0
+        , connack_return_code/0
+        , topictab/0
+        , topic/0
+        , subopts/0
+        ]).
+
+-define(CONF_DEFAULT, <<"
+exhook: { server.default: { url: \"http://127.0.0.1:9000\" } }
+">>).
+
+-define(ALL(Vars, Types, Exprs),
+        ?SETUP(fun() ->
+            State = do_setup(),
+            fun() -> do_teardown(State) end
+         end, ?FORALL(Vars, Types, Exprs))).
+
+
+%%--------------------------------------------------------------------
+%% Properties
+%%--------------------------------------------------------------------
+
+prop_client_connect() ->
+    ?ALL({ConnInfo, ConnProps},
+         {conninfo(), conn_properties()},
+       begin
+           ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]),
+           {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(),
+           Expected =
+               #{props => properties(ConnProps),
+                 conninfo => from_conninfo(ConnInfo)
+                },
+           ?assertEqual(Expected, Resp),
+           true
+       end).
+
+prop_client_connack() ->
+    ?ALL({ConnInfo, Rc, AckProps},
+         {conninfo(), connack_return_code(), ack_properties()},
+        begin
+            ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]),
+            {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(AckProps),
+                  result_code => atom_to_binary(Rc, utf8),
+                  conninfo => from_conninfo(ConnInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_authenticate() ->
+    ?ALL({ClientInfo0, AuthResult},
+         {clientinfo(), authresult()},
+        begin
+            ClientInfo = inject_magic_into(username, ClientInfo0),
+            OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult),
+            ExpectedAuthResult = case maps:get(username, ClientInfo) of
+                                     <<"baduser">> -> {error, not_authorized};
+                                     <<"gooduser">> -> ok;
+                                     <<"normaluser">> -> ok;
+                                     _ -> case AuthResult of
+                                              ok -> ok;
+                                              _ -> {error, not_authorized}
+                                          end
+                                 end,
+            ?assertEqual(ExpectedAuthResult, OutAuthResult),
+
+            {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{result => authresult_to_bool(AuthResult),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_authorize() ->
+    ?ALL({ClientInfo0, PubSub, Topic, Result},
+         {clientinfo(), oneof([publish, subscribe]),
+          topic(), oneof([allow, deny])},
+        begin
+            ClientInfo = inject_magic_into(username, ClientInfo0),
+            OutResult = emqx_hooks:run_fold(
+                          'client.authorize',
+                          [ClientInfo, PubSub, Topic],
+                          Result),
+            ExpectedOutResult = case maps:get(username, ClientInfo) of
+                                    <<"baduser">> -> deny;
+                                    <<"gooduser">> -> allow;
+                                    <<"normaluser">> -> allow;
+                                    _ -> Result
+                                 end,
+            ?assertEqual(ExpectedOutResult, OutResult),
+
+            {'on_client_authorize', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{result => aclresult_to_bool(Result),
+                  type => pubsub_to_enum(PubSub),
+                  topic => Topic,
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_connected() ->
+    ?ALL({ClientInfo, ConnInfo},
+         {clientinfo(), conninfo()},
+        begin
+            ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]),
+            {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_disconnected() ->
+    ?ALL({ClientInfo, Reason, ConnInfo},
+         {clientinfo(), shutdown_reason(), conninfo()},
+        begin
+            ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]),
+            {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{reason => stringfy(Reason),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_subscribe() ->
+    ?ALL({ClientInfo, SubProps, TopicTab},
+         {clientinfo(), sub_properties(), topictab()},
+        begin
+            ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]),
+            {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(SubProps),
+                  topic_filters => topicfilters(TopicTab),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_client_unsubscribe() ->
+    ?ALL({ClientInfo, UnSubProps, TopicTab},
+         {clientinfo(), unsub_properties(), topictab()},
+        begin
+            ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]),
+            {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{props => properties(UnSubProps),
+                  topic_filters => topicfilters(TopicTab),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_created() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]),
+            {'on_session_created', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+             ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_subscribed() ->
+    ?ALL({ClientInfo, Topic, SubOpts},
+         {clientinfo(), topic(), subopts()},
+        begin
+            ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]),
+            {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{topic => Topic,
+                  subopts => subopts(SubOpts),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_unsubscribed() ->
+    ?ALL({ClientInfo, Topic, SubOpts},
+         {clientinfo(), topic(), subopts()},
+        begin
+            ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]),
+            {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{topic => Topic,
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_resumed() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]),
+            {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_discared() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]),
+            {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_takeovered() ->
+    ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]),
+            {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_session_terminated() ->
+    ?ALL({ClientInfo, Reason, SessInfo},
+         {clientinfo(), shutdown_reason(), sessioninfo()},
+        begin
+            ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]),
+            {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(),
+            Expected =
+                #{reason => stringfy(Reason),
+                  clientinfo => from_clientinfo(ClientInfo)
+                 },
+            ?assertEqual(Expected, Resp),
+            true
+        end).
+
+prop_message_publish() ->
+    ?ALL(Msg0, message(),
+        begin
+            Msg = emqx_message:from_map(
+                    inject_magic_into(from, emqx_message:to_map(Msg0))),
+            OutMsg= emqx_hooks:run_fold('message.publish', [], Msg),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true ->
+                    ?assertEqual(Msg, OutMsg),
+                    skip;
+                _ ->
+                    ExpectedOutMsg = case emqx_message:from(Msg) of
+                                         <<"baduser">> ->
+                                             MsgMap = emqx_message:to_map(Msg),
+                                             emqx_message:from_map(
+                                               MsgMap#{qos => 0,
+                                                       topic => <<"">>,
+                                                       payload => <<"">>
+                                                      });
+                                         <<"gooduser">> = From ->
+                                             MsgMap = emqx_message:to_map(Msg),
+                                             emqx_message:from_map(
+                                               MsgMap#{topic => From,
+                                                       payload => From
+                                                      });
+                                         _ -> Msg
+                                     end,
+                    ?assertEqual(ExpectedOutMsg, OutMsg),
+
+                    {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+        end).
+
+prop_message_dropped() ->
+    ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()},
+        begin
+            ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{reason => stringfy(Reason),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+       end).
+
+prop_message_delivered() ->
+    ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
+        begin
+            ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{clientinfo => from_clientinfo(ClientInfo),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+       end).
+
+prop_message_acked() ->
+    ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
+        begin
+            ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]),
+            case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
+                true -> skip;
+                _ ->
+                    {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(),
+                    Expected =
+                        #{clientinfo => from_clientinfo(ClientInfo),
+                          message => from_message(Msg)
+                         },
+                    ?assertEqual(Expected, Resp)
+            end,
+            true
+        end).
+
+nodestr() ->
+    stringfy(node()).
+
+peerhost(#{peername := {Host, _}}) ->
+    ntoa(Host).
+
+sockport(#{sockname := {_, Port}}) ->
+    Port.
+
+%% copied from emqx_exhook
+
+ntoa({0,0,0,0,0,16#ffff,AB,CD}) ->
+    list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}));
+ntoa(IP) ->
+    list_to_binary(inet_parse:ntoa(IP)).
+
+maybe(undefined) -> <<>>;
+maybe(B) -> B.
+
+properties(undefined) -> [];
+properties(M) when is_map(M) ->
+    maps:fold(fun(K, V, Acc) ->
+        [#{name => stringfy(K),
+           value => stringfy(V)} | Acc]
+    end, [], M).
+
+topicfilters(Tfs) when is_list(Tfs) ->
+    [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs].
+
+%% @private
+stringfy(Term) when is_binary(Term) ->
+    Term;
+stringfy(Term) when is_integer(Term) ->
+    integer_to_binary(Term);
+stringfy(Term) when is_atom(Term) ->
+    atom_to_binary(Term, utf8);
+stringfy(Term) ->
+    unicode:characters_to_binary((io_lib:format("~0p", [Term]))).
+
+subopts(SubOpts) ->
+    #{qos => maps:get(qos, SubOpts, 0),
+      rh => maps:get(rh, SubOpts, 0),
+      rap => maps:get(rap, SubOpts, 0),
+      nl => maps:get(nl, SubOpts, 0),
+      share => maps:get(share, SubOpts, <<>>)
+     }.
+
+authresult_to_bool(AuthResult) ->
+    AuthResult == ok.
+
+aclresult_to_bool(Result) ->
+    Result == allow.
+
+pubsub_to_enum(publish) -> 'PUBLISH';
+pubsub_to_enum(subscribe) -> 'SUBSCRIBE'.
+
+from_conninfo(ConnInfo) ->
+    #{node => nodestr(),
+      clientid => maps:get(clientid, ConnInfo),
+      username => maybe(maps:get(username, ConnInfo, <<>>)),
+      peerhost => peerhost(ConnInfo),
+      sockport => sockport(ConnInfo),
+      proto_name => maps:get(proto_name, ConnInfo),
+      proto_ver => stringfy(maps:get(proto_ver, ConnInfo)),
+      keepalive => maps:get(keepalive, ConnInfo)
+     }.
+
+from_clientinfo(ClientInfo) ->
+    #{node => nodestr(),
+      clientid => maps:get(clientid, ClientInfo),
+      username => maybe(maps:get(username, ClientInfo, <<>>)),
+      password => maybe(maps:get(password, ClientInfo, <<>>)),
+      peerhost => ntoa(maps:get(peerhost, ClientInfo)),
+      sockport => maps:get(sockport, ClientInfo),
+      protocol => stringfy(maps:get(protocol, ClientInfo)),
+      mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)),
+      is_superuser => maps:get(is_superuser, ClientInfo, false),
+      anonymous => maps:get(anonymous, ClientInfo, true),
+      cn => maybe(maps:get(cn, ClientInfo, <<>>)),
+      dn => maybe(maps:get(dn, ClientInfo, <<>>))
+    }.
+
+from_message(Msg) ->
+    #{node => nodestr(),
+      id => emqx_guid:to_hexstr(emqx_message:id(Msg)),
+      qos => emqx_message:qos(Msg),
+      from => stringfy(emqx_message:from(Msg)),
+      topic => emqx_message:topic(Msg),
+      payload => emqx_message:payload(Msg),
+      timestamp => emqx_message:timestamp(Msg)
+     }.
+
+%%--------------------------------------------------------------------
+%% Helper
+%%--------------------------------------------------------------------
+
+do_setup() ->
+    logger:set_primary_config(#{level => warning}),
+    _ = emqx_exhook_demo_svr:start(),
+    ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
+    emqx_ct_helpers:start_apps([emqx_exhook]),
+    %% waiting first loaded event
+    {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(),
+    ok.
+
+do_teardown(_) ->
+    emqx_ct_helpers:stop_apps([emqx_exhook]),
+    %% waiting last unloaded event
+    {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(),
+    _ = emqx_exhook_demo_svr:stop(),
+    logger:set_primary_config(#{level => notice}),
+    timer:sleep(2000),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Generators
+%%--------------------------------------------------------------------
+
+conn_properties() ->
+    #{}.
+
+ack_properties() ->
+    #{}.
+
+sub_properties() ->
+    #{}.
+
+unsub_properties() ->
+    #{}.
+
+shutdown_reason() ->
+    oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]).
+
+authresult() ->
+    ?LET(RC, connack_return_code(),
+         case RC of
+             success -> ok;
+             _ -> {error, RC}
+         end).
+
+inject_magic_into(Key, Object) ->
+    case castspell() of
+        muggles -> Object;
+        Spell ->
+            Object#{Key => Spell}
+    end.
+
+castspell() ->
+    L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles],
+    lists:nth(rand:uniform(length(L)), L).

apps/emqx_gateway/etc/emqx_exhook.conf

@@ -1,15 +0,0 @@
-##====================================================================
-## EMQ X Hooks
-##====================================================================
-
-##--------------------------------------------------------------------
-## Server Address
-
-## The gRPC server url
-##
-## exhook.server.$name.url = url()
-exhook.server.default.url = "http://127.0.0.1:9000"
-
-#exhook.server.default.ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
-#exhook.server.default.ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
-#exhook.server.default.ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem"

apps/emqx_gateway/etc/emqx_gateway.conf

@@ -36,7 +36,7 @@ gateway: {
       subscribe_qos: qos0
       publish_qos: qos1
       listener.udp.1: {
-        bind: 5687
+        bind: 5683
         }
       }
 
@@ -49,7 +49,7 @@ gateway: {
       subscribe_qos: qos2
       publish_qos: coap
       listener.udp.1: {
-        bind: 5683
+        bind: 5687
         }
       }
 
@@ -127,4 +127,34 @@ gateway: {
         #listener.udp.1: {}
         #listener.dtls.1: {}
     }
+
+    lwm2m_xml_dir: "{{ platform_etc_dir }}/lwm2m_xml"
+
+    lwm2m.1: {
+
+        lifetime_min: 1s
+
+        lifetime_max: 86400s
+
+        qmode_time_windonw: 22
+
+        auto_observe: false
+
+        mountpoint: "lwm2m/%e/"
+
+        ## always | contains_object_list
+        update_msg_publish_condition: contains_object_list
+
+        translators: {
+            command: "dn/#"
+            response: "up/resp"
+            notify: "up/notify"
+            register: "up/resp"
+            update: "up/resp"
+        }
+
+        listener.udp.1 {
+            bind: 5783
+        }
+    }
 }

apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl

@@ -20,7 +20,6 @@
 -include_lib("emqx/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[PGW-Conn]").
 
 %% API
 -export([ start_link/3

apps/emqx_gateway/src/coap/emqx_coap_channel.erl

@@ -67,9 +67,9 @@
 -define(DISCONNECT_WAIT_TIME, timer:seconds(10)).
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]).
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% API
+%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
 
 info(Channel) ->
     maps:from_list(info(?INFO_KEYS, Channel)).
@@ -101,7 +101,7 @@ init(ConnInfo = #{peername := {PeerHost, _},
     ClientInfo = set_peercert_infos(
                    Peercert,
                    #{ zone => default
-                    , protocol => 'mqtt-coap'
+                    , protocol => 'coap'
                     , peerhost => PeerHost
                     , sockport => SockPort
                     , clientid => emqx_guid:to_base62(emqx_guid:gen())
@@ -132,8 +132,8 @@ auth_subscribe(Topic,
                  clientinfo := ClientInfo}) ->
     emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic).
 
-transfer_result(Result, From, Value) ->
+transfer_result(From, Value, Result) ->
-    ?TRANSFER_RESULT(Result, [out], From, Value).
+    ?TRANSFER_RESULT([out], From, Value, Result).
 
 %%--------------------------------------------------------------------
 %% Handle incoming packet
@@ -147,17 +147,17 @@ handle_in(#coap_message{method = post,
         <<>> ->
             handle_command(Msg, Channel);
         _ ->
-            call_session(Channel, received, [Msg])
+            call_session(received, [Msg], Channel)
     end;
 
 handle_in(Msg, Channel) ->
-    call_session(ensure_keepalive_timer(Channel), received, [Msg]).
+    call_session(received, [Msg], ensure_keepalive_timer(Channel)).
 
 %%--------------------------------------------------------------------
 %% Handle Delivers from broker to client
 %%--------------------------------------------------------------------
 handle_deliver(Delivers, Channel) ->
-    call_session(Channel, deliver, [Delivers]).
+    call_session(deliver, [Delivers], Channel).
 
 %%--------------------------------------------------------------------
 %% Handle timeout
@@ -165,14 +165,14 @@ handle_deliver(Delivers, Channel) ->
 handle_timeout(_, {keepalive, NewVal}, #channel{keepalive = KeepAlive} = Channel) ->
     case emqx_keepalive:check(NewVal, KeepAlive) of
         {ok, NewKeepAlive} ->
-            Channel2 = ensure_keepalive_timer(Channel, fun make_timer/4),
+            Channel2 = ensure_keepalive_timer(fun make_timer/4, Channel),
             {ok, Channel2#channel{keepalive = NewKeepAlive}};
         {error, timeout} ->
             {shutdown, timeout, Channel}
     end;
 
 handle_timeout(_, {transport, Msg}, Channel) ->
-    call_session(Channel, timeout, [Msg]);
+    call_session(timeout, [Msg], Channel);
 
 handle_timeout(_, disconnect, Channel) ->
     {shutdown, normal, Channel};
@@ -207,9 +207,9 @@ handle_info(Info, Channel) ->
 terminate(_Reason, _Channel) ->
     ok.
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% Internal functions
+%% Internal functions
-%%%===================================================================
+%%--------------------------------------------------------------------
 set_peercert_infos(NoSSL, ClientInfo)
   when NoSSL =:= nossl;
        NoSSL =:= undefined ->
@@ -232,9 +232,9 @@ make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = Timers#{Name => TRef}}.
 
 ensure_keepalive_timer(Channel) ->
-    ensure_keepalive_timer(Channel, fun ensure_timer/4).
+    ensure_keepalive_timer(fun ensure_timer/4, Channel).
 
-ensure_keepalive_timer(#channel{config = Cfg} = Channel, Fun) ->
+ensure_keepalive_timer(Fun, #channel{config = Cfg} = Channel) ->
     Interval = maps:get(heartbeat, Cfg),
     Fun(keepalive, Interval, keepalive, Channel).
 
@@ -285,9 +285,9 @@ run_conn_hooks(Input, Channel = #channel{ctx = Ctx,
                                          conninfo = ConnInfo}) ->
     ConnProps = #{},
     case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of
-                   Error = {error, _Reason} -> Error;
+        Error = {error, _Reason} -> Error;
-                   _NConnProps ->
+        _NConnProps ->
-                       {ok, Input, Channel}
+            {ok, Input, Channel}
     end.
 
 enrich_clientinfo({Queries, Msg},
@@ -339,11 +339,10 @@ ensure_connected(Channel = #channel{ctx = Ctx,
     Channel#channel{conninfo = NConnInfo}.
 
 process_connect(Channel = #channel{ctx = Ctx,
-                                   session = Session,
                                    conninfo = ConnInfo,
                                    clientinfo = ClientInfo},
                 Msg) ->
-    SessFun = fun(_,_) -> Session end,
+    SessFun = fun(_,_) -> emqx_coap_session:new() end,
     case emqx_gateway_ctx:open_session(
            Ctx,
            true,
@@ -367,14 +366,16 @@ run_hooks(Ctx, Name, Args, Acc) ->
     emqx_hooks:run_fold(Name, Args, Acc).
 
 reply(Channel, Method, Payload, Req) ->
-    call_session(Channel, reply, [Req, Method, Payload]).
+    call_session(reply, [Req, Method, Payload], Channel).
 
 ack(Channel, Method, Payload, Req) ->
-    call_session(Channel, piggyback, [Req, Method, Payload]).
+    call_session(piggyback, [Req, Method, Payload], Channel).
 
-call_session(#channel{session = Session,
+call_session(F,
-                      config = Cfg} = Channel, F, A) ->
+             A,
-    case erlang:apply(emqx_coap_session, F, [Session, Cfg | A]) of
+             #channel{session = Session,
+                      config = Cfg} = Channel) ->
+    case erlang:apply(emqx_coap_session, F, A ++ [Cfg, Session]) of
         #{out := Out,
           session := Session2} ->
             {ok, {outgoing, Out}, Channel#channel{session = Session2}};

apps/emqx_gateway/src/coap/emqx_coap_frame.erl

@@ -54,9 +54,9 @@
 -define(OPTION_PROXY_SCHEME, 39).
 -define(OPTION_SIZE1, 60).
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% API
+%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
 
 initial_parse_state(_) ->
     #{}.
@@ -64,9 +64,9 @@ initial_parse_state(_) ->
 serialize_opts() ->
     #{}.
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% serialize_pkt
+%% serialize_pkt
-%%%===================================================================
+%%--------------------------------------------------------------------
 %% empty message
 serialize_pkt(#coap_message{type = Type, method = undefined, id = MsgId}, _Opts) ->
     <<?VERSION:2, (encode_type(Type)):2, 0:4, 0:3, 0:5, MsgId:16>>;
@@ -223,9 +223,9 @@ method_to_class_code({error, proxying_not_supported}) -> {5, 05};
 method_to_class_code(Method) ->
     erlang:throw({bad_method, Method}).
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% parse
+%% parse
-%%%===================================================================
+%%--------------------------------------------------------------------
 parse(<<?VERSION:2, Type:2, 0:4, 0:3, 0:5, MsgId:16>>, ParseState) ->
     {ok,
      #coap_message{ type = decode_type(Type)
@@ -410,9 +410,9 @@ is_message(#coap_message{}) ->
 is_message(_) ->
     false.
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% Internal functions
+%% Internal functions
-%%%===================================================================
+%%--------------------------------------------------------------------
 -spec is_repeatable_option(message_option_name()) -> boolean().
 is_repeatable_option(if_match) -> true;
 is_repeatable_option(etag) -> true;

apps/emqx_gateway/src/coap/emqx_coap_impl.erl

@@ -31,6 +31,8 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+
 -dialyzer({nowarn_function, [load/0]}).
 
 %%--------------------------------------------------------------------
@@ -103,13 +105,12 @@ start_listener(InstaId, Ctx, ResourceMod, {Type, ListenOn, SocketOpts, Cfg}) ->
     Cfg2 = Cfg#{resource => ResourceMod},
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg2) of
         {ok, Pid} ->
-            io:format("Start coap ~s:~s listener on ~s successfully.~n",
+            ?ULOG("Start coap ~s:~s listener on ~s successfully.~n",
-                      [InstaId, Type, ListenOnStr]),
+                  [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
+            ?ELOG("Failed to start coap ~s:~s listener on ~s: ~0p~n",
-                      "Failed to start coap ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
-                      [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -136,13 +137,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop coap ~s:~s listener on ~s successfully.~n",
+        ok -> ?ULOG("Stop coap ~s:~s listener on ~s successfully.~n",
                         [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
+            ?ELOG("Failed to stop coap ~s:~s listener on ~s: ~0p~n",
-                      "Failed to stop coap ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
     end,
     StopRet.
 

apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl

@@ -17,7 +17,7 @@
 -module(emqx_coap_observe_res).
 
 %% API
--export([ new/0, insert/3, remove/2
+-export([ new_manager/0, insert/3, remove/2
         , res_changed/2, foreach/2]).
 -export_type([manager/0]).
 
@@ -26,6 +26,7 @@
 -type topic() :: binary().
 -type token() :: binary().
 -type seq_id() :: 0 .. ?MAX_SEQ_ID.
+
 -type res() :: #{ token := token()
                 , seq_id := seq_id()
                 }.
@@ -35,12 +36,12 @@
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
--spec new() -> manager().
+-spec new_manager() -> manager().
-new() ->
+new_manager() ->
     #{}.
 
--spec insert(manager(), topic(), token()) -> manager().
+-spec insert(topic(), token(), manager()) -> manager().
-insert(Manager, Topic, Token) ->
+insert(Topic, Token, Manager) ->
     case maps:get(Topic, Manager, undefined) of
         undefined ->
             Manager#{Topic => new_res(Token)};
@@ -48,12 +49,12 @@ insert(Manager, Topic, Token) ->
             Manager
     end.
 
--spec remove(manager(), topic()) -> manager().
+-spec remove(topic(), manager()) -> manager().
-remove(Manager, Topic) ->
+remove(Topic, Manager) ->
     maps:remove(Topic, Manager).
 
--spec res_changed(manager(), topic()) -> undefined | {token(), seq_id(), manager()}.
+-spec res_changed(topic(), manager()) -> undefined | {token(), seq_id(), manager()}.
-res_changed(Manager, Topic) ->
+res_changed(Topic, Manager) ->
     case maps:get(Topic, Manager, undefined) of
         undefined ->
             undefined;

apps/emqx_gateway/src/coap/emqx_coap_session.erl

@@ -47,52 +47,52 @@
 new() ->
     _ = emqx_misc:rand_seed(),
     #session{ transport_manager = emqx_coap_tm:new()
-            , observe_manager = emqx_coap_observe_res:new()
+            , observe_manager = emqx_coap_observe_res:new_manager()
             , next_msg_id = rand:uniform(?MAX_MESSAGE_ID)}.
 
 %%%-------------------------------------------------------------------
 %%% Process Message
 %%%-------------------------------------------------------------------
-received(Session, Cfg, #coap_message{type = ack} = Msg) ->
+received(#coap_message{type = ack} = Msg, Cfg, Session) ->
-    handle_response(Session, Cfg, Msg);
+    handle_response(Msg, Cfg, Session);
 
-received(Session, Cfg, #coap_message{type = reset} = Msg) ->
+received(#coap_message{type = reset} = Msg, Cfg, Session) ->
-    handle_response(Session, Cfg, Msg);
+    handle_response(Msg, Cfg, Session);
 
-received(Session, Cfg, #coap_message{method = Method} = Msg) when is_atom(Method) ->
+received(#coap_message{method = Method} = Msg, Cfg, Session) when is_atom(Method) ->
-    handle_request(Session, Cfg, Msg);
+    handle_request(Msg, Cfg, Session);
 
-received(Session, Cfg, Msg) ->
+received(Msg, Cfg, Session) ->
-    handle_response(Session, Cfg, Msg).
+    handle_response(Msg, Cfg, Session).
 
-reply(Session, Cfg, Req, Method) ->
+reply(Req, Method, Cfg, Session) ->
-    reply(Session, Cfg, Req, Method, <<>>).
+    reply(Req, Method, <<>>, Cfg, Session).
 
-reply(Session, Cfg, Req, Method, Payload) ->
+reply(Req, Method, Payload, Cfg, Session) ->
     Response = emqx_coap_message:response(Method, Payload, Req),
-    handle_out(Session, Cfg, Response).
+    handle_out(Response, Cfg, Session).
 
-ack(Session, Cfg, Req) ->
+ack(Req, Cfg, Session) ->
-    piggyback(Session, Cfg, Req, <<>>).
+    piggyback(Req, <<>>, Cfg, Session).
 
-piggyback(Session, Cfg, Req, Payload) ->
+piggyback(Req, Payload, Cfg, Session) ->
     Response = emqx_coap_message:ack(Req),
     Response2 = emqx_coap_message:set_payload(Payload, Response),
-    handle_out(Session, Cfg, Response2).
+    handle_out(Response2, Cfg, Session).
 
-deliver(Session, Cfg, Delivers) ->
+deliver(Delivers, Cfg, Session) ->
     Fun = fun({_, Topic, Message},
               #{out := OutAcc,
                 session := #session{observe_manager = OM,
                                     next_msg_id = MsgId} = SAcc} = Acc) ->
-                  case emqx_coap_observe_res:res_changed(OM, Topic) of
+                  case emqx_coap_observe_res:res_changed(Topic, OM) of
                       undefined ->
                           Acc;
                       {Token, SeqId, OM2} ->
                           Msg = mqtt_to_coap(Message, MsgId, Token, SeqId, Cfg),
                           SAcc2 = SAcc#session{next_msg_id = next_msg_id(MsgId),
                                                observe_manager = OM2},
-                          #{out := Out} = Result = call_transport_manager(SAcc2, Cfg, Msg, handle_out),
+                          #{out := Out} = Result = call_transport_manager(handle_out, Msg, Cfg, SAcc2),
                           Result#{out := [Out | OutAcc]}
                   end
           end,
@@ -101,35 +101,35 @@ deliver(Session, Cfg, Delivers) ->
                   session => Session},
                 Delivers).
 
-timeout(Session, Cfg, Timer) ->
+timeout(Timer, Cfg, Session) ->
-    call_transport_manager(Session, Cfg, Timer, ?FUNCTION_NAME).
+    call_transport_manager(?FUNCTION_NAME, Timer, Cfg, Session).
 
-transfer_result(Result, From, Value) ->
+transfer_result(From, Value, Result) ->
-    ?TRANSFER_RESULT(Result, [out, subscribe], From, Value).
+    ?TRANSFER_RESULT([out, subscribe], From, Value, Result).
 
 %%%-------------------------------------------------------------------
 %%% Internal functions
 %%%-------------------------------------------------------------------
-handle_request(Session, Cfg, Msg) ->
+handle_request(Msg, Cfg, Session) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-handle_response(Session, Cfg, Msg) ->
+handle_response(Msg, Cfg, Session) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-handle_out(Session, Cfg, Msg) ->
+handle_out(Msg, Cfg, Session) ->
-    call_transport_manager(Session, Cfg, Msg, ?FUNCTION_NAME).
+    call_transport_manager(?FUNCTION_NAME, Msg, Cfg, Session).
 
-call_transport_manager(#session{transport_manager = TM} = Session,
+call_transport_manager(Fun,
-                       Cfg,
                        Msg,
-                       Fun) ->
+                       Cfg,
+                       #session{transport_manager = TM} = Session) ->
     try
-        Result = emqx_coap_tm:Fun(Msg, TM, Cfg),
+        Result = emqx_coap_tm:Fun(Msg, Cfg, TM),
         {ok, _, Session2} = emqx_misc:pipeline([fun process_tm/2,
                                                 fun process_subscribe/2],
                                                Result,
                                                Session),
-        emqx_coap_channel:transfer_result(Result, session, Session2)
+        emqx_coap_channel:transfer_result(session, Session2, Result)
     catch Type:Reason:Stack ->
             ?ERROR("process transmission with, message:~p failed~n
 Type:~p,Reason:~p~n,StackTrace:~p~n", [Msg, Type, Reason, Stack]),
@@ -146,10 +146,10 @@ process_subscribe(#{subscribe := Sub}, #session{observe_manager = OM} =  Session
         undefined ->
             {ok, Session};
         {Topic, Token} ->
-            OM2 = emqx_coap_observe_res:insert(OM, Topic, Token),
+            OM2 = emqx_coap_observe_res:insert(Topic, Token, OM),
             {ok, Session#session{observe_manager = OM2}};
         Topic ->
-            OM2 = emqx_coap_observe_res:remove(OM, Topic),
+            OM2 = emqx_coap_observe_res:remove(Topic, OM),
             {ok, Session#session{observe_manager = OM2}}
     end;
 process_subscribe(_, Session) ->

apps/emqx_gateway/src/coap/emqx_coap_tm.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% transport manager
+%% the transport state machine manager
 -module(emqx_coap_tm).
 
 -export([ new/0
@@ -23,23 +23,23 @@
         , handle_out/3
         , timeout/3]).
 
--export_type([manager/0, event_result/2]).
+-export_type([manager/0, event_result/1]).
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
 -type direction() :: in | out.
--type transport_id() :: {direction(), non_neg_integer()}.
+-type state_machine_id() :: {direction(), non_neg_integer()}.
 
--record(transport, { id :: transport_id()
+-record(state_machine, { id :: state_machine_id()
-                   , state :: atom()
+                       , state :: atom()
-                   , timers :: maps:map()
+                       , timers :: maps:map()
-                   , data :: any()}).
+                       , transport :: emqx_coap_transport:transport()}).
--type transport() :: #transport{}.
+-type state_machine() :: #state_machine{}.
 
 -type message_id() :: 0 .. ?MAX_MESSAGE_ID.
 
--type manager() :: #{message_id() => transport()}.
+-type manager() :: #{message_id() => state_machine()}.
 
 -type ttimeout() :: {state_timeout, pos_integer(), any()}
                   | {stop_timeout, pos_integer()}.
@@ -47,32 +47,31 @@
 -type topic() :: binary().
 -type token() :: binary().
 -type sub_register() :: {topic(), token()} | topic().
-
+-type event_result(State) ::
--type event_result(State, Data) ::
         #{next => State,
           outgoing => emqx_coap_message(),
           timeouts => list(ttimeout()),
           has_sub  => undefined | sub_register(),
-          data => Data}.
+          transport => emqx_coap_transport:transprot()}.
 
-%%%===================================================================
+%%--------------------------------------------------------------------
-%%% API
+%% API
-%%%===================================================================
+%%--------------------------------------------------------------------
 new() ->
     #{}.
 
-handle_request(#coap_message{id = MsgId} = Msg, TM, Cfg) ->
+handle_request(#coap_message{id = MsgId} = Msg, Cfg, TM) ->
     Id = {in, MsgId},
     case maps:get(Id, TM, undefined) of
-                 undefined ->
+        undefined ->
-            Data = emqx_coap_transport:new(),
+            Transport = emqx_coap_transport:new(),
-            Transport = new_transport(Id, Data),
+            Machine = new_state_machine(Id, Transport),
-            process_event(in, Msg, TM, Transport, Cfg);
+            process_event(in, Msg, TM, Machine, Cfg);
-        TP ->
+        Machine ->
-            process_event(in, Msg, TM, TP, Cfg)
+            process_event(in, Msg, TM, Machine, Cfg)
     end.
 
-handle_response(#coap_message{type = Type, id = MsgId} = Msg, TM, Cfg) ->
+handle_response(#coap_message{type = Type, id = MsgId} = Msg, Cfg, TM) ->
     Id = {out, MsgId},
     case maps:get(Id, TM, undefined) of
         undefined ->
@@ -83,56 +82,50 @@ handle_response(#coap_message{type = Type, id = MsgId} = Msg, TM, Cfg) ->
                     #{out => #coap_message{type = reset,
                                            id = MsgId}}
             end;
-        TP ->
+        Machine ->
-            process_event(in, Msg, TM, TP, Cfg)
+            process_event(in, Msg, TM, Machine, Cfg)
     end.
 
-handle_out(#coap_message{id = MsgId} = Msg, TM, Cfg) ->
+handle_out(#coap_message{id = MsgId} = Msg, Cfg, TM) ->
     Id = {out, MsgId},
     case maps:get(Id, TM, undefined) of
         undefined ->
-            Data = emqx_coap_transport:new(),
+            Transport = emqx_coap_transport:new(),
-            Transport = new_transport(Id, Data),
+            Machine = new_state_machine(Id, Transport),
-            process_event(out, Msg, TM, Transport, Cfg);
+            process_event(out, Msg, TM, Machine, Cfg);
         _ ->
             ?WARN("Repeat sending message with id:~p~n", [Id]),
             ?EMPTY_RESULT
     end.
 
-timeout({Id, Type, Msg}, TM, Cfg) ->
+timeout({Id, Type, Msg}, Cfg, TM) ->
     case maps:get(Id, TM, undefined) of
         undefined ->
             ?EMPTY_RESULT;
-        #transport{timers = Timers} = TP ->
+        #state_machine{timers = Timers} = Machine ->
             %% maybe timer has been canceled
             case maps:is_key(Type, Timers) of
                 true ->
-                    process_event(Type, Msg, TM, TP, Cfg);
+                    process_event(Type, Msg, TM, Machine, Cfg);
                 _ ->
                     ?EMPTY_RESULT
             end
     end.
 
 %%--------------------------------------------------------------------
-%% @doc
+%% Internal functions
-%% @spec
-%% @end
 %%--------------------------------------------------------------------
-
+new_state_machine(Id, Transport) ->
-%%%===================================================================
+    #state_machine{id = Id,
-%%% Internal functions
+                   state = idle,
-%%%===================================================================
+                   timers = #{},
-new_transport(Id, Data) ->
+                   transport = Transport}.
-    #transport{id = Id,
-               state = idle,
-               timers = #{},
-               data = Data}.
 
 process_event(stop_timeout,
               _,
               TM,
-              #transport{id = Id,
+              #state_machine{id = Id,
-                         timers = Timers},
+                             timers = Timers},
               _) ->
     lists:foreach(fun({_, Ref}) ->
                           emqx_misc:cancel_timer(Ref)
@@ -143,42 +136,42 @@ process_event(stop_timeout,
 process_event(Event,
               Msg,
               TM,
-              #transport{id = Id,
+              #state_machine{id = Id,
-                         state = State,
+                             state = State,
-                         data = Data} = TP,
+                             transport = Transport} = Machine,
               Cfg) ->
-    Result = emqx_coap_transport:State(Event, Msg, Data, Cfg),
+    Result = emqx_coap_transport:State(Event, Msg, Transport, Cfg),
-    {ok, _, TP2} = emqx_misc:pipeline([fun process_state_change/2,
+    {ok, _, Machine2} = emqx_misc:pipeline([fun process_state_change/2,
-                                       fun process_data_change/2,
+                                            fun process_transport_change/2,
-                                       fun process_timeouts/2],
+                                            fun process_timeouts/2],
-                                      Result,
+                                           Result,
-                                      TP),
+                                           Machine),
-    TM2 = TM#{Id => TP2},
+    TM2 = TM#{Id => Machine2},
-    emqx_coap_session:transfer_result(Result, tm, TM2).
+    emqx_coap_session:transfer_result(tm, TM2, Result).
 
-process_state_change(#{next := Next}, TP) ->
+process_state_change(#{next := Next}, Machine) ->
-    {ok, cancel_state_timer(TP#transport{state = Next})};
+    {ok, cancel_state_timer(Machine#state_machine{state = Next})};
-process_state_change(_, TP) ->
+process_state_change(_, Machine) ->
-    {ok, TP}.
+    {ok, Machine}.
 
-cancel_state_timer(#transport{timers = Timers} = TP) ->
+cancel_state_timer(#state_machine{timers = Timers} = Machine) ->
     case maps:get(state_timer, Timers, undefined) of
         undefined ->
-            TP;
+            Machine;
         Ref ->
             _ = emqx_misc:cancel_timer(Ref),
-            TP#transport{timers = maps:remove(state_timer, Timers)}
+            Machine#state_machine{timers = maps:remove(state_timer, Timers)}
     end.
 
-process_data_change(#{data := Data}, TP) ->
+process_transport_change(#{transport := Transport}, Machine) ->
-    {ok, TP#transport{data = Data}};
+    {ok, Machine#state_machine{transport = Transport}};
-process_data_change(_, TP) ->
+process_transport_change(_, Machine) ->
-    {ok, TP}.
+    {ok, Machine}.
 
-process_timeouts(#{timeouts := []}, TP) ->
+process_timeouts(#{timeouts := []}, Machine) ->
-    {ok, TP};
+    {ok, Machine};
 process_timeouts(#{timeouts := Timeouts},
-                 #transport{id = Id, timers = Timers} = TP) ->
+                 #state_machine{id = Id, timers = Timers} = Machine) ->
     NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) ->
                                     process_timer(Id, Timer, Acc);
                                ({stop_timeout, I}, Acc) ->
@@ -186,11 +179,11 @@ process_timeouts(#{timeouts := Timeouts},
                             end,
                             Timers,
                             Timeouts),
-    {ok, TP#transport{timers = NewTimers}};
+    {ok, Machine#state_machine{timers = NewTimers}};
 
-process_timeouts(_, TP) ->
+process_timeouts(_, Machine) ->
-    {ok, TP}.
+    {ok, Machine}.
 
 process_timer(Id, {Type, Interval, Msg}, Timers) ->
-    Ref = emqx_misc:start_timer(Interval, {transport, {Id, Type, Msg}}),
+    Ref = emqx_misc:start_timer(Interval, {state_machine, {Id, Type, Msg}}),
     Timers#{Type => Ref}.

apps/emqx_gateway/src/coap/emqx_coap_transport.erl

@@ -9,21 +9,23 @@
 -define(EXCHANGE_LIFETIME, 247000).
 -define(NON_LIFETIME, 145000).
 
--record(data, { cache :: undefined | emqx_coap_message()
+-record(transport, { cache :: undefined | emqx_coap_message()
-              , retry_interval :: non_neg_integer()
+                   , retry_interval :: non_neg_integer()
-              , retry_count :: non_neg_integer()
+                   , retry_count :: non_neg_integer()
-              }).
+                   }).
 
--type data() :: #data{}.
+-type transport() :: #transport{}.
 
 -export([ new/0, idle/4, maybe_reset/4
         , maybe_resend/4, wait_ack/4, until_stop/4]).
 
--spec new() -> data().
+-export_type([transport/0]).
+
+-spec new() -> transport().
 new() ->
-    #data{cache = undefined,
+    #transport{cache = undefined,
-          retry_interval = 0,
+               retry_interval = 0,
-          retry_count = 0}.
+               retry_count = 0}.
 
 idle(in,
      #coap_message{type = non, id = MsgId, method = Method} = Msg,
@@ -50,14 +52,14 @@ idle(in,
      #coap_message{id = MsgId,
                    type = con,
                    method = Method} = Msg,
-     Data,
+     Transport,
      #{resource := Resource} = Cfg) ->
     Ret = #{next => maybe_resend,
             timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]},
     case Method of
         undefined ->
             ResetMsg = #coap_message{type = reset, id = MsgId},
-            Ret#{data => Data#data{cache = ResetMsg},
+            Ret#{transport => Transport#transport{cache = ResetMsg},
                  out  => ResetMsg};
         _ ->
             {RetMsg, SubInfo} =
@@ -72,7 +74,7 @@ idle(in,
                 end,
             RetMsg2 = RetMsg#coap_message{type = ack},
             Ret#{out => RetMsg2,
-                 data => Data#data{cache = RetMsg2},
+                 transport => Transport#transport{cache = RetMsg2},
                  subscribe => SubInfo}
     end;
 
@@ -81,11 +83,11 @@ idle(out, #coap_message{type = non} = Msg, _, _) ->
       out => Msg,
       timeouts => [{stop_timeout, ?NON_LIFETIME}]};
 
-idle(out, Msg, Data, _) ->
+idle(out, Msg, Transport, _) ->
     _ = emqx_misc:rand_seed(),
     Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR),
     #{next => wait_ack,
-      data => Data#data{cache = Msg},
+      transport => Transport#transport{cache = Msg},
       out => Msg,
       timeouts => [ {state_timeout, Timeout, ack_timeout}
                   , {stop_timeout, ?EXCHANGE_LIFETIME}]}.
@@ -99,7 +101,7 @@ maybe_reset(in, Message, _, _) ->
     end,
     ?EMPTY_RESULT.
 
-maybe_resend(in, _, _, #data{cache = Cache}) ->
+maybe_resend(in, _, _, #transport{cache = Cache}) ->
     #{out => Cache}.
 
 wait_ack(in, #coap_message{type = Type}, _, _) ->
@@ -115,14 +117,14 @@ wait_ack(in, #coap_message{type = Type}, _, _) ->
 wait_ack(state_timeout,
          ack_timeout,
          _,
-         #data{cache = Msg,
+         #transport{cache = Msg,
-               retry_interval = Timeout,
+                    retry_interval = Timeout,
-               retry_count = Count} =Data) ->
+                    retry_count = Count} =Transport) ->
     case Count < ?MAX_RETRANSMIT of
         true ->
             Timeout2 = Timeout * 2,
-            #{data => Data#data{retry_interval = Timeout2,
+            #{transport => Transport#transport{retry_interval = Timeout2,
-                                retry_count = Count + 1},
+                                               retry_count = Count + 1},
               out => Msg,
               timeouts => [{state_timeout, Timeout2, ack_timeout}]};
         _ ->

apps/emqx_gateway/src/coap/include/emqx_coap.hrl

@@ -23,7 +23,7 @@
 -define(MAXIMUM_MAX_AGE, 4294967295).
 
 -define(EMPTY_RESULT, #{}).
--define(TRANSFER_RESULT(R1, Keys, From, Value),
+-define(TRANSFER_RESULT(Keys, From, Value, R1),
         begin
             R2 = maps:with(Keys, R1),
             R2#{From => Value}

apps/emqx_gateway/src/coap/resources/emqx_coap_mqtt_resource.erl

@@ -22,7 +22,6 @@
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-RES]").
 
 -export([ init/1
         , stop/1

apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_resource.erl

@@ -22,7 +22,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-PS-RES]").
 
 -export([ init/1
         , stop/1

apps/emqx_gateway/src/coap/resources/emqx_coap_pubsub_topics.erl

@@ -21,7 +21,6 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
 
--logger_header("[CoAP-PS-TOPICS]").
 
 -export([ start_link/0
         , stop/1

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -3,7 +3,7 @@
   {vsn, "0.1.0"},
   {registered, []},
   {mod, {emqx_gateway_app, []}},
-  {applications, [kernel, stdlib, grpc]},
+  {applications, [kernel, stdlib, grpc, lwm2m_coap]},
   {env, []},
   {modules, []},
   {licenses, ["Apache 2.0"]},

apps/emqx_gateway/src/emqx_gateway_app.erl

@@ -20,7 +20,6 @@
 
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[Gateway]").
 
 -export([start/2, stop/1]).
 
@@ -45,7 +44,8 @@ load_default_gateway_applications() ->
 
 gateway_type_searching() ->
     %% FIXME: Hardcoded apps
-    [emqx_stomp_impl, emqx_sn_impl, emqx_exproto_impl, emqx_coap_impl].
+    [emqx_stomp_impl, emqx_sn_impl, emqx_exproto_impl,
+     emqx_coap_impl, emqx_lwm2m_impl].
 
 load(Mod) ->
     try
@@ -82,10 +82,14 @@ create_gateway_by_default([{Type, Name, Confs}|More]) ->
     create_gateway_by_default(More).
 
 zipped_confs() ->
-    All = maps:to_list(emqx_config:get([gateway])),
+    All = maps:to_list(
+            maps:without(exclude_options(), emqx_config:get([gateway]))),
     lists:append(lists:foldr(
       fun({Type, Gws}, Acc) ->
         {Names, Confs} = lists:unzip(maps:to_list(Gws)),
         Types = [ Type || _ <- lists:seq(1, length(Names))],
         [lists:zip3(Types, Names, Confs) | Acc]
       end, [], All)).
+
+exclude_options() ->
+   [lwm2m_xml_dir].

apps/emqx_gateway/src/emqx_gateway_cm.erl

@@ -26,7 +26,6 @@
 -include("include/emqx_gateway.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[PGW-CM]").
 
 %% APIs
 -export([start_link/1]).

apps/emqx_gateway/src/emqx_gateway_cm_registry.erl

@@ -19,7 +19,6 @@
 
 -behaviour(gen_server).
 
--logger_header("[PGW-CM-Registy]").
 
 -export([start_link/1]).
 

apps/emqx_gateway/src/emqx_gateway_ctx.erl

@@ -19,7 +19,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header(["PGW-Ctx"]).
 
 %% @doc The running context for a Connection/Channel process.
 %%

apps/emqx_gateway/src/emqx_gateway_insta_sup.erl

@@ -21,7 +21,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Insta-Sup]").
 
 %% APIs
 -export([ start_link/3

apps/emqx_gateway/src/emqx_gateway_metrics.erl

@@ -20,7 +20,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Metrics]").
 
 %% APIs
 -export([start_link/1]).

apps/emqx_gateway/src/emqx_gateway_registry.erl

@@ -19,7 +19,6 @@
 
 -include("include/emqx_gateway.hrl").
 
--logger_header("[PGW-Registry]").
 
 -behavior(gen_server).
 

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -34,8 +34,10 @@ structs() -> ["gateway"].
 fields("gateway") ->
     [{stomp, t(ref(stomp))},
      {mqttsn, t(ref(mqttsn))},
-     {exproto, t(ref(exproto))},
+     {coap, t(ref(coap))},
-     {coap, t(ref(coap))}
+     {lwm2m, t(ref(lwm2m))},
+     {lwm2m_xml_dir, t(string())},
+     {exproto, t(ref(exproto))}
     ];
 
 fields(stomp) ->
@@ -74,6 +76,21 @@ fields(mqttsn_predefined) ->
     , {topic, t(string())}
     ];
 
+fields(lwm2m) ->
+    [{"$id", t(ref(lwm2m_structs))}
+    ];
+
+fields(lwm2m_structs) ->
+    [ {lifetime_min, t(duration())}
+    , {lifetime_max, t(duration())}
+    , {qmode_time_windonw, t(integer())}
+    , {auto_observe, t(boolean())}
+    , {mountpoint, t(string())}
+    , {update_msg_publish_condition, t(union([always, contains_object_list]))}
+    , {translators, t(ref(translators))}
+    , {listener, t(ref(udp_listener_group))}
+    ];
+
 fields(exproto) ->
     [{"$id", t(ref(exproto_structs))}];
 
@@ -100,6 +117,9 @@ fields(clientinfo_override) ->
     , {clientid, t(string())}
     ];
 
+fields(translators) ->
+    [{"$name", t(string())}];
+
 fields(udp_listener_group) ->
     [ {udp, t(ref(udp_listener))}
     , {dtls, t(ref(dtls_listener))}

apps/emqx_gateway/src/exhook/prop_exhook_hooks.erl

@@ -1,531 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(prop_exhook_hooks).
-
-% -include_lib("proper/include/proper.hrl").
-% -include_lib("eunit/include/eunit.hrl").
-
-% -import(emqx_ct_proper_types,
-%         [ conninfo/0
-%         , clientinfo/0
-%         , sessioninfo/0
-%         , message/0
-%         , connack_return_code/0
-%         , topictab/0
-%         , topic/0
-%         , subopts/0
-%         ]).
-
-% -define(ALL(Vars, Types, Exprs),
-%         ?SETUP(fun() ->
-%             State = do_setup(),
-%             fun() -> do_teardown(State) end
-%          end, ?FORALL(Vars, Types, Exprs))).
-
-% %%--------------------------------------------------------------------
-% %% Properties
-% %%--------------------------------------------------------------------
-
-% prop_client_connect() ->
-%     ?ALL({ConnInfo, ConnProps},
-%          {conninfo(), conn_properties()},
-%        begin
-%            ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]),
-%            {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(),
-%            Expected =
-%                #{props => properties(ConnProps),
-%                  conninfo => from_conninfo(ConnInfo)
-%                 },
-%            ?assertEqual(Expected, Resp),
-%            true
-%        end).
-
-% prop_client_connack() ->
-%     ?ALL({ConnInfo, Rc, AckProps},
-%          {conninfo(), connack_return_code(), ack_properties()},
-%         begin
-%             ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]),
-%             {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(AckProps),
-%                   result_code => atom_to_binary(Rc, utf8),
-%                   conninfo => from_conninfo(ConnInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_authenticate() ->
-%     ?ALL({ClientInfo0, AuthResult},
-%          {clientinfo(), authresult()},
-%         begin
-%             ClientInfo = inject_magic_into(username, ClientInfo0),
-%             OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult),
-%             ExpectedAuthResult = case maps:get(username, ClientInfo) of
-%                                      <<"baduser">> ->
-%                                          AuthResult#{
-%                                            auth_result => not_authorized,
-%                                            anonymous => false};
-%                                      <<"gooduser">> ->
-%                                          AuthResult#{
-%                                            auth_result => success,
-%                                            anonymous => false};
-%                                      <<"normaluser">> ->
-%                                          AuthResult#{
-%                                            auth_result => success,
-%                                            anonymous => false};
-%                                      _ ->
-%                                          case maps:get(auth_result, AuthResult) of
-%                                              success ->
-%                                                  #{auth_result => success,
-%                                                    anonymous => false};
-%                                              _ ->
-%                                                  #{auth_result => not_authorized,
-%                                                    anonymous => false}
-%                                          end
-%                                  end,
-%             ?assertEqual(ExpectedAuthResult, OutAuthResult),
-
-%             {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{result => authresult_to_bool(AuthResult),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_authorize() ->
-%     ?ALL({ClientInfo0, PubSub, Topic, Result},
-%          {clientinfo(), oneof([publish, subscribe]),
-%           topic(), oneof([allow, deny])},
-%         begin
-%             ClientInfo = inject_magic_into(username, ClientInfo0),
-%             OutResult = emqx_hooks:run_fold(
-%                           'client.authorize',
-%                           [ClientInfo, PubSub, Topic],
-%                           Result),
-%             ExpectedOutResult = case maps:get(username, ClientInfo) of
-%                                     <<"baduser">> -> deny;
-%                                     <<"gooduser">> -> allow;
-%                                     <<"normaluser">> -> allow;
-%                                     _ -> Result
-%                                  end,
-%             ?assertEqual(ExpectedOutResult, OutResult),
-
-%             {'on_client_authorize', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{result => authzresult_to_bool(Result),
-%                   type => pubsub_to_enum(PubSub),
-%                   topic => Topic,
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_connected() ->
-%     ?ALL({ClientInfo, ConnInfo},
-%          {clientinfo(), conninfo()},
-%         begin
-%             ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]),
-%             {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_disconnected() ->
-%     ?ALL({ClientInfo, Reason, ConnInfo},
-%          {clientinfo(), shutdown_reason(), conninfo()},
-%         begin
-%             ok = emqx_hooks:run('client.disconnected', [ClientInfo, Reason, ConnInfo]),
-%             {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{reason => stringfy(Reason),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_subscribe() ->
-%     ?ALL({ClientInfo, SubProps, TopicTab},
-%          {clientinfo(), sub_properties(), topictab()},
-%         begin
-%             ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]),
-%             {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(SubProps),
-%                   topic_filters => topicfilters(TopicTab),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_client_unsubscribe() ->
-%     ?ALL({ClientInfo, UnSubProps, TopicTab},
-%          {clientinfo(), unsub_properties(), topictab()},
-%         begin
-%             ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]),
-%             {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{props => properties(UnSubProps),
-%                   topic_filters => topicfilters(TopicTab),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_created() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]),
-%             {'on_session_created', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%              ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_subscribed() ->
-%     ?ALL({ClientInfo, Topic, SubOpts},
-%          {clientinfo(), topic(), subopts()},
-%         begin
-%             ok = emqx_hooks:run('session.subscribed', [ClientInfo, Topic, SubOpts]),
-%             {'on_session_subscribed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{topic => Topic,
-%                   subopts => subopts(SubOpts),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_unsubscribed() ->
-%     ?ALL({ClientInfo, Topic, SubOpts},
-%          {clientinfo(), topic(), subopts()},
-%         begin
-%             ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, SubOpts]),
-%             {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{topic => Topic,
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_resumed() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]),
-%             {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_discared() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]),
-%             {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_takeovered() ->
-%     ?ALL({ClientInfo, SessInfo}, {clientinfo(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]),
-%             {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_session_terminated() ->
-%     ?ALL({ClientInfo, Reason, SessInfo},
-%          {clientinfo(), shutdown_reason(), sessioninfo()},
-%         begin
-%             ok = emqx_hooks:run('session.terminated', [ClientInfo, Reason, SessInfo]),
-%             {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(),
-%             Expected =
-%                 #{reason => stringfy(Reason),
-%                   clientinfo => from_clientinfo(ClientInfo)
-%                  },
-%             ?assertEqual(Expected, Resp),
-%             true
-%         end).
-
-% prop_message_publish() ->
-%     ?ALL(Msg0, message(),
-%         begin
-%             Msg = emqx_message:from_map(
-%                     inject_magic_into(from, emqx_message:to_map(Msg0))),
-%             OutMsg= emqx_hooks:run_fold('message.publish', [], Msg),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true ->
-%                     ?assertEqual(Msg, OutMsg),
-%                     skip;
-%                 _ ->
-%                     ExpectedOutMsg = case emqx_message:from(Msg) of
-%                                          <<"baduser">> ->
-%                                              MsgMap = emqx_message:to_map(Msg),
-%                                              emqx_message:from_map(
-%                                                MsgMap#{qos => 0,
-%                                                        topic => <<"">>,
-%                                                        payload => <<"">>
-%                                                       });
-%                                          <<"gooduser">> = From ->
-%                                              MsgMap = emqx_message:to_map(Msg),
-%                                              emqx_message:from_map(
-%                                                MsgMap#{topic => From,
-%                                                        payload => From
-%                                                       });
-%                                          _ -> Msg
-%                                      end,
-%                     ?assertEqual(ExpectedOutMsg, OutMsg),
-
-%                     {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%         end).
-
-% prop_message_dropped() ->
-%     ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()},
-%         begin
-%             ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{reason => stringfy(Reason),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%        end).
-
-% prop_message_delivered() ->
-%     ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
-%         begin
-%             ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{clientinfo => from_clientinfo(ClientInfo),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%        end).
-
-% prop_message_acked() ->
-%     ?ALL({ClientInfo, Msg}, {clientinfo(), message()},
-%         begin
-%             ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]),
-%             case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of
-%                 true -> skip;
-%                 _ ->
-%                     {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(),
-%                     Expected =
-%                         #{clientinfo => from_clientinfo(ClientInfo),
-%                           message => from_message(Msg)
-%                          },
-%                     ?assertEqual(Expected, Resp)
-%             end,
-%             true
-%         end).
-
-% nodestr() ->
-%     stringfy(node()).
-
-% peerhost(#{peername := {Host, _}}) ->
-%     ntoa(Host).
-
-% sockport(#{sockname := {_, Port}}) ->
-%     Port.
-
-% %% copied from emqx_exhook
-
-% ntoa({0,0,0,0,0,16#ffff,AB,CD}) ->
-%     list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}));
-% ntoa(IP) ->
-%     list_to_binary(inet_parse:ntoa(IP)).
-
-% maybe(undefined) -> <<>>;
-% maybe(B) -> B.
-
-% properties(undefined) -> [];
-% properties(M) when is_map(M) ->
-%     maps:fold(fun(K, V, Acc) ->
-%         [#{name => stringfy(K),
-%            value => stringfy(V)} | Acc]
-%     end, [], M).
-
-% topicfilters(Tfs) when is_list(Tfs) ->
-%     [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs].
-
-% %% @private
-% stringfy(Term) when is_binary(Term) ->
-%     Term;
-% stringfy(Term) when is_integer(Term) ->
-%     integer_to_binary(Term);
-% stringfy(Term) when is_atom(Term) ->
-%     atom_to_binary(Term, utf8);
-% stringfy(Term) ->
-%     unicode:characters_to_binary((io_lib:format("~0p", [Term]))).
-
-% subopts(SubOpts) ->
-%     #{qos => maps:get(qos, SubOpts, 0),
-%       rh => maps:get(rh, SubOpts, 0),
-%       rap => maps:get(rap, SubOpts, 0),
-%       nl => maps:get(nl, SubOpts, 0),
-%       share => maps:get(share, SubOpts, <<>>)
-%      }.
-
-% authresult_to_bool(AuthResult) ->
-%     maps:get(auth_result, AuthResult, undefined) == success.
-
-% authzresult_to_bool(Result) ->
-%     Result == allow.
-
-% pubsub_to_enum(publish) -> 'PUBLISH';
-% pubsub_to_enum(subscribe) -> 'SUBSCRIBE'.
-
-% from_conninfo(ConnInfo) ->
-%     #{node => nodestr(),
-%       clientid => maps:get(clientid, ConnInfo),
-%       username => maybe(maps:get(username, ConnInfo, <<>>)),
-%       peerhost => peerhost(ConnInfo),
-%       sockport => sockport(ConnInfo),
-%       proto_name => maps:get(proto_name, ConnInfo),
-%       proto_ver => stringfy(maps:get(proto_ver, ConnInfo)),
-%       keepalive => maps:get(keepalive, ConnInfo)
-%      }.
-
-% from_clientinfo(ClientInfo) ->
-%     #{node => nodestr(),
-%       clientid => maps:get(clientid, ClientInfo),
-%       username => maybe(maps:get(username, ClientInfo, <<>>)),
-%       password => maybe(maps:get(password, ClientInfo, <<>>)),
-%       peerhost => ntoa(maps:get(peerhost, ClientInfo)),
-%       sockport => maps:get(sockport, ClientInfo),
-%       protocol => stringfy(maps:get(protocol, ClientInfo)),
-%       mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)),
-%       is_superuser => maps:get(is_superuser, ClientInfo, false),
-%       anonymous => maps:get(anonymous, ClientInfo, true),
-%       cn => maybe(maps:get(cn, ClientInfo, <<>>)),
-%       dn => maybe(maps:get(dn, ClientInfo, <<>>))
-%     }.
-
-% from_message(Msg) ->
-%     #{node => nodestr(),
-%       id => emqx_guid:to_hexstr(emqx_message:id(Msg)),
-%       qos => emqx_message:qos(Msg),
-%       from => stringfy(emqx_message:from(Msg)),
-%       topic => emqx_message:topic(Msg),
-%       payload => emqx_message:payload(Msg),
-%       timestamp => emqx_message:timestamp(Msg)
-%      }.
-
-% %%--------------------------------------------------------------------
-% %% Helper
-% %%--------------------------------------------------------------------
-
-% do_setup() ->
-%     logger:set_primary_config(#{level => warning}),
-%     _ = emqx_exhook_demo_svr:start(),
-%     emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
-%     %% waiting first loaded event
-%     {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(),
-%     ok.
-
-% do_teardown(_) ->
-%     emqx_ct_helpers:stop_apps([emqx_exhook]),
-%     %% waiting last unloaded event
-%     {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(),
-%     _ = emqx_exhook_demo_svr:stop(),
-%     logger:set_primary_config(#{level => notice}),
-%     timer:sleep(2000),
-%     ok.
-
-% set_special_cfgs(emqx) ->
-%     application:set_env(emqx, allow_anonymous, false),
-%     application:set_env(emqx, enable_authz_cache, false),
-%     application:set_env(emqx, modules_loaded_file, undefined),
-%     application:set_env(emqx, plugins_loaded_file,
-%                         emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins"));
-% set_special_cfgs(emqx_exhook) ->
-%     ok.
-
-% %%--------------------------------------------------------------------
-% %% Generators
-% %%--------------------------------------------------------------------
-
-% conn_properties() ->
-%     #{}.
-
-% ack_properties() ->
-%     #{}.
-
-% sub_properties() ->
-%     #{}.
-
-% unsub_properties() ->
-%     #{}.
-
-% shutdown_reason() ->
-%     oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]).
-
-% authresult() ->
-%     ?LET(RC, connack_return_code(), #{auth_result => RC}).
-
-% inject_magic_into(Key, Object) ->
-%     case castspell() of
-%         muggles -> Object;
-%         Spell ->
-%             Object#{Key => Spell}
-%     end.
-
-% castspell() ->
-%     L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles],
-%     lists:nth(rand:uniform(length(L)), L).

apps/emqx_gateway/src/exhook/test/emqx_exhook_SUITE.erl

@@ -1,97 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_exhook_SUITE).
-
-% -compile(export_all).
-% -compile(nowarn_export_all).
-
-
-% -include_lib("eunit/include/eunit.hrl").
-% -include_lib("common_test/include/ct.hrl").
-
-% %%--------------------------------------------------------------------
-% %% Setups
-% %%--------------------------------------------------------------------
-
-% all() -> emqx_ct:all(?MODULE).
-
-% init_per_suite(Cfg) ->
-%     _ = emqx_exhook_demo_svr:start(),
-%     emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1),
-%     Cfg.
-
-% end_per_suite(_Cfg) ->
-%     emqx_ct_helpers:stop_apps([emqx_exhook]),
-%     emqx_exhook_demo_svr:stop().
-
-% set_special_cfgs(emqx) ->
-%     application:set_env(emqx, allow_anonymous, false),
-%     application:set_env(emqx, enable_authz_cache, false),
-%     application:set_env(emqx, plugins_loaded_file, undefined),
-%     application:set_env(emqx, modules_loaded_file, undefined);
-% set_special_cfgs(emqx_exhook) ->
-%     ok.
-
-% %%--------------------------------------------------------------------
-% %% Test cases
-% %%--------------------------------------------------------------------
-
-% t_noserver_nohook(_) ->
-%     emqx_exhook:disable(default),
-%     ?assertEqual([], ets:tab2list(emqx_hooks)),
-
-%     Opts = proplists:get_value(
-%              default,
-%              application:get_env(emqx_exhook, servers, [])
-%             ),
-%     ok = emqx_exhook:enable(default, Opts),
-%     ?assertNotEqual([], ets:tab2list(emqx_hooks)).
-
-% t_cli_list(_) ->
-%     meck_print(),
-%     ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]]
-%                 , emqx_exhook_cli:cli(["server", "list"])
-%                 ),
-%     unmeck_print().
-
-% t_cli_enable_disable(_) ->
-%     meck_print(),
-%     ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
-%     ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
-%     ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])),
-
-%     ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
-%     ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
-%     unmeck_print().
-
-% t_cli_stats(_) ->
-%     meck_print(),
-%     _ = emqx_exhook_cli:cli(["server", "stats"]),
-%     _ = emqx_exhook_cli:cli(x),
-%     unmeck_print().
-
-% %%--------------------------------------------------------------------
-% %% Utils
-% %%--------------------------------------------------------------------
-
-% meck_print() ->
-%     meck:new(emqx_ctl, [passthrough, no_history, no_link]),
-%     meck:expect(emqx_ctl, print, fun(_) -> ok end),
-%     meck:expect(emqx_ctl, print, fun(_, Args) -> Args end).
-
-% unmeck_print() ->
-%     meck:unload(emqx_ctl).

apps/emqx_gateway/src/exhook/test/emqx_exhook_demo_svr.erl

@@ -1,339 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_exhook_demo_svr).
-
-% -behavior(emqx_exhook_v_1_hook_provider_bhvr).
-
-% %%
-% -export([ start/0
-%         , stop/0
-%         , take/0
-%         , in/1
-%         ]).
-
-% %% gRPC server HookProvider callbacks
-% -export([ on_provider_loaded/2
-%         , on_provider_unloaded/2
-%         , on_client_connect/2
-%         , on_client_connack/2
-%         , on_client_connected/2
-%         , on_client_disconnected/2
-%         , on_client_authenticate/2
-%         , on_client_authorize/2
-%         , on_client_subscribe/2
-%         , on_client_unsubscribe/2
-%         , on_session_created/2
-%         , on_session_subscribed/2
-%         , on_session_unsubscribed/2
-%         , on_session_resumed/2
-%         , on_session_discarded/2
-%         , on_session_takeovered/2
-%         , on_session_terminated/2
-%         , on_message_publish/2
-%         , on_message_delivered/2
-%         , on_message_dropped/2
-%         , on_message_acked/2
-%         ]).
-
-% -define(PORT, 9000).
-% -define(NAME, ?MODULE).
-
-% %%--------------------------------------------------------------------
-% %% Server APIs
-% %%--------------------------------------------------------------------
-
-% start() ->
-%     Pid = spawn(fun mngr_main/0),
-%     register(?MODULE, Pid),
-%     {ok, Pid}.
-
-% stop() ->
-%     grpc:stop_server(?NAME),
-%     ?MODULE ! stop.
-
-% take() ->
-%     ?MODULE ! {take, self()},
-%     receive {value, V} -> V
-%     after 5000 -> error(timeout) end.
-
-% in({FunName, Req}) ->
-%     ?MODULE ! {in, FunName, Req}.
-
-% mngr_main() ->
-%     application:ensure_all_started(grpc),
-%     Services = #{protos => [emqx_exhook_pb],
-%                  services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
-%                 },
-%     Options = [],
-%     Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
-%     mngr_loop([Svr, queue:new(), queue:new()]).
-
-% mngr_loop([Svr, Q, Takes]) ->
-%     receive
-%         {in, FunName, Req} ->
-%             {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
-%             mngr_loop([Svr, NQ1, NQ2]);
-%         {take, From} ->
-%             {NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
-%             mngr_loop([Svr, NQ1, NQ2]);
-%         stop ->
-%             exit(normal)
-%     end.
-
-% reply(Q1, Q2) ->
-%     case queue:len(Q1) =:= 0 orelse
-%          queue:len(Q2) =:= 0 of
-%         true -> {Q1, Q2};
-%         _ ->
-%             {{value, {Name, V}}, NQ1} = queue:out(Q1),
-%             {{value, From}, NQ2} = queue:out(Q2),
-%             From ! {value, {Name, V}},
-%             {NQ1, NQ2}
-%     end.
-
-% %%--------------------------------------------------------------------
-% %% callbacks
-% %%--------------------------------------------------------------------
-
-% -spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-
-% on_provider_loaded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{hooks => [
-%                      #{name => <<"client.connect">>},
-%                      #{name => <<"client.connack">>},
-%                      #{name => <<"client.connected">>},
-%                      #{name => <<"client.disconnected">>},
-%                      #{name => <<"client.authenticate">>},
-%                      #{name => <<"client.authorize">>},
-%                      #{name => <<"client.subscribe">>},
-%                      #{name => <<"client.unsubscribe">>},
-%                      #{name => <<"session.created">>},
-%                      #{name => <<"session.subscribed">>},
-%                      #{name => <<"session.unsubscribed">>},
-%                      #{name => <<"session.resumed">>},
-%                      #{name => <<"session.discarded">>},
-%                      #{name => <<"session.takeovered">>},
-%                      #{name => <<"session.terminated">>},
-%                      #{name => <<"message.publish">>},
-%                      #{name => <<"message.delivered">>},
-%                      #{name => <<"message.acked">>},
-%                      #{name => <<"message.dropped">>}]}, Md}.
-% -spec on_provider_unloaded(emqx_exhook_pb:provider_unloaded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_provider_unloaded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connect(emqx_exhook_pb:client_connect_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connect(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connack(emqx_exhook_pb:client_connack_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connack(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_connected(emqx_exhook_pb:client_connected_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_connected(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_disconnected(emqx_exhook_pb:client_disconnected_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_disconnected(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case Username of
-%         <<"baduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, false}}, Md};
-%         <<"gooduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, true}}, Md};
-%         <<"normaluser">> ->
-%             {ok, #{type => 'CONTINUE',
-%                    value => {bool_result, true}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_client_authorize(emqx_exhook_pb:client_authorize_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_authorize(#{clientinfo := #{username := Username}} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case Username of
-%         <<"baduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, false}}, Md};
-%         <<"gooduser">> ->
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {bool_result, true}}, Md};
-%         <<"normaluser">> ->
-%             {ok, #{type => 'CONTINUE',
-%                    value => {bool_result, true}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_subscribe(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_client_unsubscribe(emqx_exhook_pb:client_unsubscribe_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_client_unsubscribe(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_created(emqx_exhook_pb:session_created_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_created(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_subscribed(emqx_exhook_pb:session_subscribed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_subscribed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_unsubscribed(emqx_exhook_pb:session_unsubscribed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_unsubscribed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_resumed(emqx_exhook_pb:session_resumed_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_resumed(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_discarded(emqx_exhook_pb:session_discarded_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_discarded(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_takeovered(emqx_exhook_pb:session_takeovered_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_takeovered(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_session_terminated(emqx_exhook_pb:session_terminated_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_session_terminated(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_publish(#{message := #{from := From} = Msg} = Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     %% some cases for testing
-%     case From of
-%         <<"baduser">> ->
-%             NMsg = Msg#{qos => 0,
-%                         topic => <<"">>,
-%                         payload => <<"">>
-%                        },
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {message, NMsg}}, Md};
-%         <<"gooduser">> ->
-%             NMsg = Msg#{topic => From,
-%                         payload => From},
-%             {ok, #{type => 'STOP_AND_RETURN',
-%                    value => {message, NMsg}}, Md};
-%         _ ->
-%             {ok, #{type => 'IGNORE'}, Md}
-%     end.
-
-% -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_delivered(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_dropped(emqx_exhook_pb:message_dropped_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_dropped(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.
-
-% -spec on_message_acked(emqx_exhook_pb:message_acked_request(), grpc:metadata())
-%     -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()}
-%      | {error, grpc_cowboy_h:error_response()}.
-% on_message_acked(Req, Md) ->
-%     ?MODULE:in({?FUNCTION_NAME, Req}),
-%     %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]),
-%     {ok, #{}, Md}.

apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl

@@ -21,7 +21,6 @@
 -include_lib("emqx/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto Channel]").
 
 -export([ info/1
         , info/2

apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl

@@ -21,7 +21,6 @@
 
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto gClient]").
 
 %% APIs
 -export([async_call/3]).

apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl

@@ -22,7 +22,6 @@
 -include("src/exproto/include/emqx_exproto.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--logger_header("[ExProto gServer]").
 
 -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)).
 

apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl

@@ -32,6 +32,8 @@
         , on_insta_destroy/3
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -64,8 +66,7 @@ start_grpc_server(InstaId, Options = #{bind := ListenOn}) ->
                       [{ssl_options, SslOpts}]
               end,
     _ = grpc:start_server(InstaId, ListenOn, Services, SvrOptions),
-    io:format("Start ~s gRPC server on ~p successfully.~n",
+    ?ULOG("Start ~s gRPC server on ~p successfully.~n", [InstaId, ListenOn]).
-               [InstaId, ListenOn]).
 
 start_grpc_client_channel(InstaId, Options = #{address := UriStr}) ->
     UriMap = uri_string:parse(UriStr),
@@ -146,13 +147,12 @@ start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) ->
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of
         {ok, Pid} ->
-            io:format("Start exproto ~s:~s listener on ~s successfully.~n",
+            ?ULOG("Start exproto ~s:~s listener on ~s successfully.~n",
                       [InstaId, Type, ListenOnStr]),
             Pid;
         {error, Reason} ->
-            io:format(standard_error,
+            ?ELOG("Failed to start exproto ~s:~s listener on ~s: ~0p~n",
-                      "Failed to start exproto ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason]),
-                      [InstaId, Type, ListenOnStr, Reason]),
             throw({badconf, Reason})
     end.
 
@@ -204,13 +204,11 @@ stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) ->
     StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg),
     ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
     case StopRet of
-        ok -> io:format("Stop exproto ~s:~s listener on ~s successfully.~n",
+        ok -> ?ULOG("Stop exproto ~s:~s listener on ~s successfully.~n",
-                        [InstaId, Type, ListenOnStr]);
+                    [InstaId, Type, ListenOnStr]);
         {error, Reason} ->
-            io:format(standard_error,
+            ?ELOG("Failed to stop exproto ~s:~s listener on ~s: ~0p~n",
-                      "Failed to stop exproto ~s:~s listener on ~s: ~0p~n",
+                  [InstaId, Type, ListenOnStr, Reason])
-                      [InstaId, Type, ListenOnStr, Reason]
-                     )
     end,
     StopRet.
 

apps/emqx_gateway/src/lwm2m/emqx_lwm2m.app.src

@@ -1,7 +0,0 @@
-{application,emqx_lwm2m,
-             [{description,"EMQ X LwM2M Gateway"},
-              {vsn, "4.3.2"}, % strict semver, bump manually!
-              {modules,[]},
-              {registered,[emqx_lwm2m_sup]},
-              {applications,[kernel,stdlib,lwm2m_coap]},
-              {mod,{emqx_lwm2m_app,[]}}]}.