Merge remote-tracking branch 'origin/release-v43' into 1004-fix-wildcard-redispatch-for-shared-subs

.github/workflows/apps_version_check.yaml

@@ -28,7 +28,9 @@ jobs:
         if: endsWith(github.repository, 'enterprise')
         run: ./scripts/update-appup.sh emqx-ee --check
       - name: Check apps version
-        run: ./scripts/apps-version-check.sh
+        run: ./scripts/check-apps-vsn.sh
+      - name: Check chart versions
+        run: ./scripts/check-chart-vsn.sh
       - uses: actions/upload-artifact@v3.1.0
         if: failure()
         with:

.github/workflows/build_slim_packages.yaml

@@ -49,12 +49,7 @@ jobs:
     - name: make xref
       run: make xref
     - name: make dialyzer
-      run: |
+      run: make dialyzer
-        # TODO: once dialyzer warnings are fixed in 4.4+, we may
-        # enforce this.
-        if [[ "$(./pkg-vsn.sh)" =~ 4.3.* ]]; then
-          make dialyzer
-        fi
     - name: build zip packages
       run: make ${EMQX_NAME}-zip
     - name: build deb/rpm packages
@@ -64,7 +59,7 @@ jobs:
       with:
         name: rebar3.crashdump
         path: ./rebar3.crashdump
-    - name: pakcages test
+    - name: packages test
       run: |
         export CODE_PATH=$GITHUB_WORKSPACE
         .ci/build_packages/tests.sh
@@ -102,18 +97,27 @@ jobs:
       id: cache
       with:
         path: ~/.kerl/${{ matrix.erl_otp }}
-        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
+        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
       env:
         KERL_BUILD_BACKEND: git
         OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
       run: |
         kerl update releases
         kerl build ${{ matrix.erl_otp }}
         kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
     - name: build
+      env:
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
       run: |
         . $HOME/.kerl/${{ matrix.erl_otp }}/activate
         make ensure-rebar3
@@ -129,7 +133,10 @@ jobs:
         pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
         gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        # test with a spaces in path
+        mv ./emqx "./emqx  home/"
+        cd "./emqx  home/"
+        ./bin/emqx start || cat log/erlang.log.1
         ready='no'
         for i in {1..10}; do
           if curl -fs 127.0.0.1:18083 > /dev/null; then
@@ -140,12 +147,13 @@ jobs:
         done
         if [ "$ready" != "yes" ]; then
           echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
+          cat log/erlang.log.1
           exit 1
         fi
-        ./emqx/bin/emqx_ctl status
+        ./bin/emqx_ctl status
-        ./emqx/bin/emqx stop
+        ./bin/emqx stop
-        rm -rf emqx
+        cd ..
+        rm -rf "emqx  home"
     - uses: actions/upload-artifact@v2
       with:
         name: macos

.github/workflows/run_test_cases.yaml

@@ -5,6 +5,8 @@ on:
     tags:
       - v*
       - e*
+    branches:
+      - 'main-v4.[0-9]?'
   pull_request:
 
 jobs:
@@ -26,6 +28,7 @@ jobs:
     run_common_test:
         runs-on: ${{ matrix.runs-on }}
         strategy:
+          fail-fast: false
           matrix:
             runs-on:
               - aws-amd64
@@ -110,6 +113,7 @@ jobs:
         - name: run cover
           run: |
             printenv > .env
+            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
             docker exec -i erlang bash -c "make cover"
             docker exec --env-file .env -i erlang bash -c "make coveralls"
         - name: cat rebar.crashdump

CHANGES-4.3.md

@@ -10,6 +10,44 @@ File format:
 - One list item per change topic
   Change log ends with a list of GitHub PRs
 
+## v4.3.22
+
+### Minor changes
+
+## v4.3.21
+
+### Enhancements
+
+- TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005).
+  New config `listener.ssl.$NAME.hibernate_after` to hibernate TLS connection process after idling.
+  Hibernation can reduce RAM usage significantly, but may cost more CPU.
+  This configuration is by default disabled.
+  Our preliminary test shows a 50% of RAM usage decline when configured to '5s'.
+
+- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
+  Eliminate uncertainty that the buffer size is set by OS default.
+
+- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
+
+- Added a test to prevent a last will testament message to be
+  published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)
+
+### Bug fixes
+
+- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908)
+
+- Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071)
+  In this change, it also included more changes in redis client:
+  - Improve redis connection error logging [eredis:19](https://github.com/emqx/eredis/pull/19).
+    Also added support for eredis to accept an anonymous function as password instead of
+    passing around plaintext args which may get dumpped to crash logs (hard to predict where).
+    This change also added `format_status` callback for `gen_server` states which hold plaintext
+    password so the process termination log and `sys:get_status` will print '******' instead of
+    the password to console.
+  - Avoid pool name clashing [eredis_cluster#22](https://github.com/emqx/eredis_cluster/pull/22)
+    Same `format_status` callback is added here too for `gen_server`s which hold password in
+    their state.
+
 ## v4.3.20
 
 ### Bug fixes

Makefile

@@ -90,7 +90,7 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 clean: $(PROFILES:%=clean-%)
 $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
-		rm rebar.lock \
+		rm -f rebar.lock; \
 		rm -rf _build/$(@:clean-%=%)/rel; \
 		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
 		$(FIND) _build/$(@:clean-%=%) -type l -delete; \
@@ -114,8 +114,9 @@ $(PROFILES:%=deps-%): $(REBAR) get-dashboard
 	@rm -f rebar.lock
 
 .PHONY: xref
-xref: $(REBAR)
+xref: $(REBAR) $(REL_PROFILES:%=%-rel)
 	@$(REBAR) as check xref
+	@scripts/xref-check.escript
 
 .PHONY: dialyzer
 dialyzer: $(REBAR)

apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.3.6"}, % strict semver, bump manually!
+  {vsn, "4.3.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},

apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src

@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.5",
+  [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
     [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
@@ -12,7 +13,8 @@
      {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
    {<<".*">>,[]}],
-  [{"4.3.5",
+  [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
     [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
    {"4.3.4",

apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl

@@ -200,7 +200,7 @@ do_verify(JwsCompacted, [Jwk|More]) ->
     end.
 
 check_claims(Claims) ->
-    Now = os:system_time(seconds),
+    Now = erlang:system_time(seconds),
     Checker = [{<<"exp">>, with_num_value(
                              fun(ExpireTime) -> Now < ExpireTime end)},
                {<<"iat">>, with_num_value(

apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl

@@ -75,7 +75,7 @@ t_check_auth(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client1">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>),
     ct:pal("Jwt: ~p~n", [Jwt]),
 
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -101,7 +101,7 @@ t_check_nbf(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client1">>},
                 {username, <<"plain">>},
-                {nbf, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {nbf, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     ct:pal("Jwt: ~p~n", [Jwt]),
 
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -114,7 +114,7 @@ t_check_iat(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client1">>},
                 {username, <<"plain">>},
-                {iat, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {iat, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     ct:pal("Jwt: ~p~n", [Jwt]),
 
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -147,7 +147,7 @@ t_check_auth_str_exp(init, _Config) ->
     application:unset_env(emqx_auth_jwt, verify_claims).
 t_check_auth_str_exp(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
-    Exp = integer_to_binary(os:system_time(seconds) + 3),
+    Exp = integer_to_binary(erlang:system_time(seconds) + 3),
 
     Jwt0 = sign([{clientid, <<"client1">>},
                  {username, <<"plain">>},
@@ -167,7 +167,7 @@ t_check_auth_str_exp(_Config) ->
     ct:pal("Auth result: ~p~n", [Result1]),
     ?assertMatch({error, _}, Result1),
 
-    Exp2 = float_to_binary(os:system_time(seconds) + 3.5),
+    Exp2 = float_to_binary(erlang:system_time(seconds) + 3.5),
 
     Jwt2 = sign([{clientid, <<"client1">>},
                  {username, <<"plain">>},
@@ -182,7 +182,7 @@ t_check_auth_float_exp(init, _Config) ->
     application:unset_env(emqx_auth_jwt, verify_claims).
 t_check_auth_float_exp(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
-    Exp = os:system_time(seconds) + 3.5,
+    Exp = erlang:system_time(seconds) + 3.5,
 
     Jwt0 = sign([{clientid, <<"client1">>},
                  {username, <<"plain">>},
@@ -209,7 +209,7 @@ t_check_claims(_Config) ->
     Jwt = sign([{client_id, <<"client1">>},
                 {username, <<"plain">>},
                 {sub, value},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -225,7 +225,7 @@ t_check_claims_clientid(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -241,7 +241,7 @@ t_check_claims_username(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{client_id, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -257,7 +257,7 @@ t_check_claims_kid_in_header(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}],
+                {exp, erlang:system_time(seconds) + 3}],
                #{<<"alg">> => <<"HS256">>,
                  <<"kid">> => <<"a_kid_str">>}, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -298,7 +298,7 @@ t_check_jwt_acl(_Config) ->
                 {sub, value},
                 {acl, [{sub, [<<"a/b">>]},
                        {pub, [<<"c/d">>]}]},
-                {exp, os:system_time(seconds) + 10}],
+                {exp, erlang:system_time(seconds) + 10}],
                <<"HS256">>,
                <<"emqxsecret">>),
 
@@ -338,7 +338,7 @@ t_check_jwt_acl_no_recs(_Config) ->
                 {username, <<"plain">>},
                 {sub, value},
                 {acl, []},
-                {exp, os:system_time(seconds) + 10}],
+                {exp, erlang:system_time(seconds) + 10}],
                <<"HS256">>,
                <<"emqxsecret">>),
 
@@ -361,7 +361,7 @@ t_check_jwt_acl_no_acl_claim(_Config) ->
     Jwt = sign([{client_id, <<"client1">>},
                 {username, <<"plain">>},
                 {sub, value},
-                {exp, os:system_time(seconds) + 10}],
+                {exp, erlang:system_time(seconds) + 10}],
                <<"HS256">>,
                <<"emqxsecret">>),
 
@@ -423,7 +423,7 @@ t_check_jwt_acl_expire(_Config) ->
                 {username, <<"plain">>},
                 {sub, value},
                 {acl, [{sub, [<<"a/b">>]}]},
-                {exp, os:system_time(seconds) + 1}],
+                {exp, erlang:system_time(seconds) + 1}],
                <<"HS256">>,
                <<"emqxsecret">>),
 

apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(export_all).
 
 -include("emqx_auth_mnesia.hrl").
+-include_lib("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -77,15 +78,37 @@ init_per_testcase_migration(_, Config) ->
     emqx_acl_mnesia_migrator:migrate_records(),
     Config.
 
+init_per_testcase_other(t_last_will_testament_message_check_acl, Config) ->
+    OriginalACLNoMatch = application:get_env(emqx, acl_nomatch),
+    application:set_env(emqx, acl_nomatch, deny),
+    emqx_mod_acl_internal:unload([]),
+    %% deny all for this client
+    ClientID = <<"lwt_client">>,
+    ok = emqx_acl_mnesia_db:add_acl({clientid, ClientID}, <<"#">>, pubsub, deny),
+    [ {original_acl_nomatch, OriginalACLNoMatch}
+    , {clientid, ClientID}
+    | Config];
+init_per_testcase_other(_TestCase, Config) ->
+    Config.
+
 init_per_testcase(Case, Config) ->
     PerTestInitializers = [
         fun init_per_testcase_clean/2,
         fun init_per_testcase_migration/2,
-        fun init_per_testcase_emqx_hook/2
+        fun init_per_testcase_emqx_hook/2,
+        fun init_per_testcase_other/2
     ],
     lists:foldl(fun(Init, Conf) -> Init(Case, Conf) end, Config, PerTestInitializers).
 
-end_per_testcase(_, Config) ->
+end_per_testcase(t_last_will_testament_message_check_acl, Config) ->
+    emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5),
+    case ?config(original_acl_nomatch, Config) of
+        {ok, Original} -> application:set_env(emqx, acl_nomatch, Original);
+        _ -> ok
+    end,
+    emqx_mod_acl_internal:load([]),
+    ok;
+end_per_testcase(_TestCase, Config) ->
     emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5),
     Config.
 
@@ -464,6 +487,35 @@ t_rest_api(_Config) ->
     {ok, Res3} = request_http_rest_list(["$all"]),
     ?assertMatch([], get_http_data(Res3)).
 
+%% asserts that we check ACL for the LWT topic before publishing the
+%% LWT.
+t_last_will_testament_message_check_acl(Config) ->
+    ClientID = ?config(clientid, Config),
+    {ok, C} = emqtt:start_link([
+        {clientid, ClientID},
+        {will_topic, <<"$SYS/lwt">>},
+        {will_payload, <<"should not be published">>}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqx:subscribe(<<"$SYS/lwt">>),
+    unlink(C),
+    ok = snabbkaffe:start_trace(),
+    {true, {ok, _}} =
+        ?wait_async_action(
+        exit(C, kill),
+        #{?snk_kind := last_will_testament_publish_denied},
+        1_000
+    ),
+    ok = snabbkaffe:stop(),
+
+    receive
+        {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} ->
+            error(lwt_should_not_be_published_to_forbidden_topic)
+    after 1_000 ->
+        ok
+    end,
+
+    ok.
 
 create_conflicting_records() ->
     Records = [

apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl

@@ -21,6 +21,9 @@
 -include("emqx_auth_mnesia.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -import(emqx_ct_http, [ request_api/3
                       , request_api/5
@@ -74,6 +77,20 @@ set_default(ClientId, UserName, Pwd, HashType) ->
     application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]),
     application:set_env(emqx_auth_mnesia, password_hash, HashType),
     ok.
+
+init_per_testcase(t_will_message_connection_denied, Config) ->
+    emqx_zone:set_env(external, allow_anonymous, false),
+    Config;
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_will_message_connection_denied, _Config) ->
+    emqx_zone:unset_env(external, allow_anonymous),
+    application:stop(emqx_auth_mnesia),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -390,6 +407,48 @@ t_password_hash(_) ->
     application:stop(emqx_auth_mnesia),
     ok = application:start(emqx_auth_mnesia).
 
+t_will_message_connection_denied(Config) when is_list(Config) ->
+    ClientId = Username = <<"subscriber">>,
+    Password = <<"p">>,
+    application:stop(emqx_auth_mnesia),
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia]),
+    ok = emqx_auth_mnesia_cli:add_user({clientid, ClientId}, Password),
+
+    {ok, Subscriber} = emqtt:start_link([
+        {clientid, ClientId},
+        {password, Password}
+    ]),
+    {ok, _} = emqtt:connect(Subscriber),
+    {ok, _, [?RC_SUCCESS]} = emqtt:subscribe(Subscriber, <<"lwt">>),
+
+    process_flag(trap_exit, true),
+
+    {ok, Publisher} = emqtt:start_link([
+        {clientid, <<"publisher">>},
+        {will_topic, <<"lwt">>},
+        {will_payload, <<"should not be published">>}
+    ]),
+    snabbkaffe:start_trace(),
+    ?wait_async_action(
+        {error, _} = emqtt:connect(Publisher),
+        #{?snk_kind := channel_terminated}
+    ),
+    snabbkaffe:stop(),
+
+    timer:sleep(1000),
+
+    receive
+        {publish, #{
+            topic := <<"lwt">>,
+            payload := <<"should not be published">>
+        }} ->
+            ct:fail("should not publish will message")
+    after 0 ->
+        ok
+    end,
+
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -52,10 +52,14 @@ all() ->
 init_per_suite(Cfg) ->
     emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1),
     init_mongo_data(),
+    %% avoid inter-suite flakiness
+    ok = emqx_mod_acl_internal:unload([]),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_mongo_data(),
+    %% avoid inter-suite flakiness
+    ok = emqx_mod_acl_internal:load([]),
     emqx_ct_helpers:stop_apps([emqx_auth_mongo]).
 
 set_special_confs(emqx) ->

apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl

@@ -147,4 +147,3 @@ safe_get(K, ClientInfo) ->
 bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
 bin(B) when is_binary(B) -> B;
 bin(X) -> X.
-

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook,
  [{description, "EMQ X Extension for Hook"},
-  {vsn, "4.3.7"},
+  {vsn, "4.3.8"},
   {modules, []},
   {registered, []},
   {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook.appup.src

@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[5-6]">>,
+  [{"4.3.7",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
     [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]},
@@ -15,7 +16,8 @@
      {update,emqx_exhook_mngr,{advanced,["4.3.4"]}}]},
    {<<"4\\.3\\.[0-3]">>,[{restart_application,emqx_exhook}]},
    {<<".*">>,[]}],
-  [{<<"4\\.3\\.[5-6]">>,
+  [{"4.3.7",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
     [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]},

apps/emqx_exhook/src/emqx_exhook_server.erl

@@ -46,7 +46,7 @@
           channel :: pid(),
           %% Registered hook names and options
           hookspec :: #{hookpoint() => map()},
-          %% Metrcis name prefix
+          %% Metrics name prefix
           prefix :: list()
        }).
 
@@ -75,8 +75,6 @@
 
 -export_type([server/0]).
 
--dialyzer({nowarn_function, [inc_metrics/2]}).
-
 %%--------------------------------------------------------------------
 %% Load/Unload APIs
 %%--------------------------------------------------------------------
@@ -249,10 +247,6 @@ call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts,
     end.
 
 %% @private
-inc_metrics(IncFun, Name) when is_function(IncFun) ->
-    %% BACKW: e4.2.0-e4.2.2
-    {env, [Prefix|_]} = erlang:fun_info(IncFun, env),
-    inc_metrics(Prefix, Name);
 inc_metrics(Prefix, Name) when is_list(Prefix) ->
     emqx_metrics:inc(list_to_atom(Prefix ++ atom_to_list(Name))).
 

apps/emqx_exproto/src/emqx_exproto.app.src

@@ -1,6 +1,6 @@
 {application, emqx_exproto,
  [{description, "EMQ X Extension for Protocol"},
-  {vsn, "4.3.11"}, %% 4.3.3 is used by ee
+  {vsn, "4.3.12"}, %% 4.3.3 is used by ee
   {modules, []},
   {registered, []},
   {mod, {emqx_exproto_app, []}},

apps/emqx_exproto/src/emqx_exproto.appup.src

@@ -1,9 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.10",[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
+  [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.10",
+    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-8]">>,
     [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},
@@ -15,9 +19,13 @@
      {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.10",[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
+  [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.10",
+    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-8]">>,
     [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},

apps/emqx_exproto/src/emqx_exproto_conn.erl

@@ -91,14 +91,6 @@
 
 -define(ENABLED(X), (X =/= undefined)).
 
--dialyzer({nowarn_function,
-           [ system_terminate/4
-           , handle_call/3
-           , handle_msg/2
-           , shutdown/3
-           , stop/3
-           ]}).
-
 %% udp
 start_link(Socket = {udp, _SockPid, _Sock}, Peername, Options) ->
     Args = [self(), Socket, Peername, Options],
@@ -501,6 +493,7 @@ terminate(Reason, State = #state{channel = Channel}) ->
 system_continue(Parent, _Debug, State) ->
     recvloop(Parent, State).
 
+-spec system_terminate(atom(), term(), term(), state()) -> no_return().
 system_terminate(Reason, _Parent, _Debug, State) ->
     terminate(Reason, State).
 

apps/emqx_management/src/emqx_management.app.src

@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.17"}, % strict semver, bump manually!
+  {vsn, "4.3.18"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -185,7 +185,6 @@ confs_to_binary(Confs) ->
 
 -endif.
 
--dialyzer([{nowarn_function, [import_rules/1, import_rule/1]}]).
 import_rule(#{<<"id">> := RuleId,
               <<"rawsql">> := RawSQL,
               <<"actions">> := Actions,
@@ -537,7 +536,6 @@ do_import_acl_mnesia(Acls) ->
                   end, Acls).
 
 -ifdef(EMQX_ENTERPRISE).
--dialyzer({nowarn_function, [import_modules/1]}).
 import_modules(Modules) ->
     case ets:info(emqx_modules) of
         undefined ->

apps/emqx_psk_file/include/emqx_psk_file.hrl

@@ -0,0 +1,25 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_PSK_FILE).
+-define(EMQX_PSK_FILE, true).
+
+-define(PSK_FILE_TAB, emqx_psk_file).
+
+-record(psk_entry, {psk_id :: binary(),
+                    psk_str :: binary()}).
+
+-endif.

apps/emqx_psk_file/src/emqx_psk_file.app.src

@@ -1,6 +1,6 @@
 {application, emqx_psk_file,
  [{description,"EMQX PSK Plugin from File"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.1"}, % strict semver, bump manually!
   {modules,[]},
   {registered,[emqx_psk_file_sup]},
   {applications,[kernel,stdlib]},

apps/emqx_psk_file/src/emqx_psk_file.appup.src

@@ -0,0 +1,10 @@
+%% -*- mode: erlang -*-
+{VSN,
+  [{"4.3.0",
+    [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]},
+     {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]}
+  ],
+  [{"4.3.0",
+    [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]},
+     {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]}
+  ]}.

apps/emqx_psk_file/src/emqx_psk_file.erl

@@ -16,6 +16,7 @@
 
 -module(emqx_psk_file).
 
+-include("emqx_psk_file.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -26,15 +27,10 @@
 %% Hooks functions
 -export([on_psk_lookup/2]).
 
--define(TAB, ?MODULE).
 -define(LF, 10).
 
--record(psk_entry, {psk_id :: binary(),
-                    psk_str :: binary()}).
-
 %% Called when the plugin application start
 load(Env) ->
-    _ = ets:new(?TAB, [set, named_table, {keypos, #psk_entry.psk_id}]),
     {ok, PskFile} = file:open(get_value(path, Env), [read, raw, binary, read_ahead]),
     preload_psks(PskFile, bin(get_value(delimiter, Env))),
     _ = file:close(PskFile),
@@ -45,7 +41,7 @@ unload() ->
     emqx:unhook('tls_handshake.psk_lookup', fun ?MODULE:on_psk_lookup/2).
 
 on_psk_lookup(ClientPSKID, UserState) ->
-    case ets:lookup(?TAB, ClientPSKID) of
+    case ets:lookup(?PSK_FILE_TAB, ClientPSKID) of
         [#psk_entry{psk_str = PskStr}] ->
             {stop, PskStr};
         [] ->
@@ -57,7 +53,9 @@ preload_psks(FileHandler, Delimiter) ->
         {ok, Line} ->
             case binary:split(Line, Delimiter) of
                 [Key, Rem] ->
-                    ets:insert(?TAB, #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}),
+                    ets:insert(
+                      ?PSK_FILE_TAB,
+                      #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}),
                     preload_psks(FileHandler, Delimiter);
                 [Line] ->
                     ?LOG(warning, "[~p] - Invalid line: ~p, delimiter: ~p", [?MODULE, Line, Delimiter])

apps/emqx_psk_file/src/emqx_psk_file_sup.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_psk_file_sup).
 
+-include("emqx_psk_file.hrl").
+
 -behaviour(supervisor).
 
 %% API
@@ -25,8 +27,11 @@
 -export([init/1]).
 
 start_link() ->
+    _ = ets:new(
+          ?PSK_FILE_TAB,
+          [set, named_table, public, {keypos, #psk_entry.psk_id}]
+         ),
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
     {ok, { {one_for_one, 0, 1}, []} }.
-

apps/emqx_retainer/src/emqx_retainer.app.src

@@ -1,6 +1,6 @@
 {application, emqx_retainer,
  [{description, "EMQ X Retainer"},
-  {vsn, "4.3.4"}, % strict semver, bump manually!
+  {vsn, "4.3.5"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_retainer_sup]},
   {applications, [kernel,stdlib]},

apps/emqx_retainer/src/emqx_retainer_sup.erl

@@ -34,6 +34,23 @@ init([Env]) ->
              type     => worker,
              modules  => [emqx_retainer]} || not is_managed_by_modules()]}}.
 
+-ifdef(EMQX_ENTERPRISE).
+
+is_managed_by_modules() ->
+    try
+        case supervisor:get_childspec(emqx_modules_sup, emqx_retainer) of
+            {ok, _} -> true;
+            _ -> false
+        end
+    catch
+        exit : {noproc, _} ->
+            false
+    end.
+
+-else.
+
 is_managed_by_modules() ->
     %% always false for opensource edition
     false.
+
+-endif.

apps/emqx_retainer/test/emqx_retainer_ct_helper.erl

@@ -20,7 +20,9 @@
 -export([ensure_start/0, ensure_stop/0]).
 -ifdef(EMQX_ENTERPRISE).
 ensure_start() ->
+    %% for enterprise edition, retainer is started by modules
     application:stop(emqx_modules),
+    ensure_stop(),
     init_conf(),
     emqx_ct_helpers:start_apps([emqx_retainer]),
     ok.
@@ -29,6 +31,7 @@ ensure_start() ->
 
 ensure_start() ->
     init_conf(),
+    ensure_stop(),
     emqx_ct_helpers:start_apps([emqx_retainer]),
     ok.
 

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.15"}, % strict semver, bump manually!
+  {vsn, "4.3.16"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,24 +1,36 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.14",
+  [{"4.3.15",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
+   {"4.3.14",
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.3.13",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -26,7 +38,8 @@
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@@ -34,7 +47,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -184,24 +198,36 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.14",
+  [{"4.3.15",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
+   {"4.3.14",
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.3.13",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
@@ -209,7 +235,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -217,7 +244,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -207,7 +207,7 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) ->
         Reason -> {error, Reason}
     end.
 
--spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()}}).
+-spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()} | term()}).
 update_rule(Params = #{id := RuleId}) ->
     case emqx_rule_registry:get_rule(RuleId) of
         {ok, Rule0} ->
@@ -336,7 +336,6 @@ start_resource(ResId) ->
             {error, {resource_not_found, ResId}}
     end.
 
--dialyzer([{nowarn_function, test_resource/1}]).
 -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}).
 test_resource(#{type := Type} = Params) ->
     case emqx_rule_registry:find_resource_type(Type) of

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -207,12 +207,6 @@
             <<"Bad Arguments: ", R0/binary>>
         end).
 
--dialyzer({nowarn_function, [create_rule/2,
-                             test_rule_sql/1,
-                             do_create_rule/1,
-                             update_rule/2
-                             ]}).
-
 %%------------------------------------------------------------------------------
 %% Rules API
 %%------------------------------------------------------------------------------

apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl

@@ -96,7 +96,6 @@ unload() ->
 %%-----------------------------------------------------------------------------
 %% 'rules' command
 %%-----------------------------------------------------------------------------
--dialyzer([{nowarn_function, [rules/1]}]).
 rules(["list"]) ->
     print_all(emqx_rule_registry:get_rules_ordered_by_ts());
 

apps/emqx_rule_engine/src/emqx_rule_runtime.erl

@@ -180,7 +180,6 @@ select_and_collect([Field|More], Input, {Output, LastKV}) ->
         {nested_put(Key, Val, Output), LastKV}).
 
 %% Filter each item got from FOREACH
--dialyzer({nowarn_function, filter_collection/4}).
 filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) ->
     lists:filtermap(
         fun(Item) ->

apps/emqx_rule_engine/src/emqx_rule_sqltester.erl

@@ -20,16 +20,6 @@
 -export([ test/1
         ]).
 
-%% Dialyzer gives up on the generated code.
-%% probably due to stack depth, or inlines.
--dialyzer({nowarn_function, [test/1,
-                             test_rule/4,
-                             flatten/1,
-                             sql_test_action/0,
-                             fill_default_values/2,
-                             envs_examp/1
-                             ]}).
-
 -spec(test(#{}) -> {ok, map() | list()} | {error, term()}).
 test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) ->
     {ok, Select} = emqx_rule_sqlparser:parse_select(Sql),

apps/emqx_stomp/src/emqx_stomp.app.src

@@ -1,6 +1,6 @@
 {application, emqx_stomp,
  [{description, "EMQ X Stomp Protocol Plugin"},
-  {vsn, "4.3.5"}, % strict semver, bump manually!
+  {vsn, "4.3.6"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_stomp_sup]},
   {applications, [kernel,stdlib]},

apps/emqx_stomp/src/emqx_stomp.appup.src

@@ -1,6 +1,10 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.4",
+  [{"4.3.5",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
@@ -19,7 +23,10 @@
     [{restart_application,emqx_stomp},
      {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]},
    {<<".*">>,[]}],
-  [{"4.3.4",
+  [{"4.3.5",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.3",

apps/emqx_stomp/src/emqx_stomp_connection.erl

@@ -91,13 +91,6 @@
 
 -define(ENABLED(X), (X =/= undefined)).
 
--dialyzer({nowarn_function, [ ensure_stats_timer/2
-                            ]}).
-
--dialyzer({no_return, [ init/1
-                      , init_state/3
-                      ]}).
-
 start_link(Transport, Sock, ProtoEnv) ->
     {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Sock, ProtoEnv]])}.
 
@@ -143,6 +136,7 @@ call(Pid, Req) ->
 call(Pid, Req, Timeout) ->
     gen_server:call(Pid, Req, Timeout).
 
+-spec init([term()]) -> no_return().
 init([Transport, RawSocket, ProtoEnv]) ->
     case Transport:wait(RawSocket) of
         {ok, Socket} ->
@@ -152,6 +146,7 @@ init([Transport, RawSocket, ProtoEnv]) ->
             exit_on_sock_error(Reason)
     end.
 
+-spec init_state(module(), port(), [proplists:property()]) -> no_return().
 init_state(Transport, Socket, ProtoEnv) ->
     {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
     {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),

apps/emqx_stomp/src/emqx_stomp_protocol.erl

@@ -50,9 +50,29 @@
         , handle_recv_nack_frame/2
         ]).
 
+-type stomp_conninfo() :: #{socktype := emqx_types:socktype(),
+                            sockname := emqx_types:peername(),
+                            peername := emqx_types:peername(),
+                            peercert := nossl | undefined | esockd_peercert:peercert(),
+                            conn_mod := module(),
+                            proto_name => binary(),
+                            proto_ver => emqx_types:ver(),
+                            clean_start => boolean(),
+                            clientid => emqx_types:clientid(),
+                            username => emqx_types:username(),
+                            conn_props => emqx_types:properties(),
+                            connected => boolean(),
+                            connected_at => undefined | non_neg_integer(),
+                            disconnected_at => non_neg_integer(),
+                            keepalive => undefined | 0..16#FFFF,
+                            receive_maximum => non_neg_integer(),
+                            expiry_interval => non_neg_integer(),
+                            atom() => term()
+                           }.
+
 -record(pstate, {
           %% Stomp ConnInfo
-          conninfo :: emqx_types:conninfo(),
+          conninfo :: stomp_conninfo(),
           %% Stomp ClientInfo
           clientinfo :: emqx_types:clientinfo(),
           %% Stomp Heartbeats
@@ -104,10 +124,6 @@
                      awaiting_rel_max
                     ]).
 
--dialyzer({nowarn_function, [ check_acl/3
-                            , init/2
-                            ]}).
-
 -type(pstate() :: #pstate{}).
 
 %% @doc Init protocol
@@ -687,12 +703,8 @@ backoff({Cx, Cy}) ->
 parse_topic_filters(TopicFilters) ->
     lists:map(fun emqx_topic:parse/1, TopicFilters).
 
-check_acl(PubSub, Topic, State = #pstate{clientinfo = ClientInfo}) ->
+check_acl(PubSub, Topic, #pstate{clientinfo = ClientInfo}) ->
-    case is_acl_enabled(State) andalso
+    emqx_access_control:check_acl(ClientInfo, PubSub, Topic).
-         emqx_access_control:check_acl(ClientInfo, PubSub, Topic) of
-        false -> allow;
-        Res   -> Res
-    end.
 
 do_subscribe(TopicFilter, SubOpts,
              State = #pstate{clientinfo = ClientInfo, subscriptions = Subs}) ->
@@ -728,10 +740,6 @@ find_sub_by_id(Id, Subs) ->
         [Sub | _] -> Sub
     end.
 
-is_acl_enabled(_) ->
-    %% TODO: configs from somewhere
-    true.
-
 %% automaticly fill the next sub-id and ack if sub-id is absent
 enrich_sub_opts(SubOpts0, Subs) ->
     SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0),

bin/emqx

@@ -481,6 +481,16 @@ case "$1" in
         ;;
 esac
 
+if [ "$IS_BOOT_COMMAND" = 'no' ]; then
+    # for non-boot commands, inspect vm.<time>.args for node name
+    # shellcheck disable=SC2012,SC2086
+    LATEST_VM_ARGS_FILE="$(ls -t "$RUNNER_DATA_DIR"/configs/vm.*.args 2>/dev/null | head -1)"
+    if [ -z "$LATEST_VM_ARGS_FILE" ]; then
+        echoerr "There is no vm.*.args config file found in '$RUNNER_DATA_DIR/configs/'"
+        echoerr "Please make sure the node is initialized (started for at least once)"
+        exit 1
+    fi
+fi
 
 if [ -z "$NAME_ARG" ]; then
     NODENAME="${EMQX_NODE_NAME:-}"
@@ -488,15 +498,7 @@ if [ -z "$NAME_ARG" ]; then
     [ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}"
     if [ -z "$NODENAME" ]; then
         if [ "$IS_BOOT_COMMAND" = 'no' ]; then
-            # for non-boot commands, inspect vm.<time>.args for node name
+            NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS_FILE" | awk '{print $2}')"
-            # shellcheck disable=SC2012,SC2086
-            LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args 2>/dev/null | head -1)"
-            if [ -z "$LATEST_VM_ARGS" ]; then
-                echoerr "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
-                echoerr "Please make sure the node is initialized (started for at least once)"
-                exit 1
-            fi
-            NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
         else
             # for boot commands, inspect emqx.conf for node name
             NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name)
@@ -531,13 +533,7 @@ if [ -z "$COOKIE" ]; then
     if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
         COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie)
     else
-        # shellcheck disable=SC2012,SC2086
+        COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS_FILE" | awk '{print $2}')"
-        LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
-        if [ -z "$LATEST_VM_ARGS" ]; then
-            echo "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
-            exit 1
-        fi
-        COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
     fi
 fi
 

build

@@ -165,7 +165,41 @@ make_zip() {
     ## try to be portable for zip packages.
     ## for DEB and RPM packages the dependencies are resoved by yum and apt
     cp_dyn_libs "${tard}/emqx"
+    case "$SYSTEM" in
+        macos*)
+            # if the flag to sign macos binaries is set, but developer certificate
+            # or certificate password is not configured, reset the flag
+            # could happen, for example, when people submit PR from a fork, in this
+            # case they cannot access secrets
+            if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
+                      ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
+                            "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
+                echo "Apple developer certificate is not configured, skip signing"
+                APPLE_SIGN_BINARIES=0
+            fi
+            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
+                ./scripts/macos-sign-binaries.sh "${tard}/emqx"
+            fi
+            ## create zip after change dir
+            ## to avoid creating an extra level of 'emqx' dir in the .zip file
             (cd "${tard}" && zip -qr - emqx) > "${zipball}"
+            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
+                # notarize the package
+                # if fails, you can check what went wrong with this command:
+                # xcrun notarytool log --apple-id <apple id> \
+                    #   --apple-id <apple id> \
+                    #   --password <apple id password>
+                    #   --team-id <apple team id> <submission-id>
+                xcrun notarytool submit \
+                      --apple-id "${APPLE_ID}" \
+                      --password "${APPLE_ID_PASSWORD}" \
+                      --team-id "${APPLE_TEAM_ID}" "${zipball}" --wait
+            fi
+            ;;
+        *)
+            (cd "${tard}" && zip -qr - emqx) > "${zipball}"
+            ;;
+    esac
 }
 
 ## This function builds the default docker image based on alpine:3.14 (by default)

deploy/charts/emqx/Chart.yaml

@@ -13,8 +13,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 4.3.20
+version: 4.3.21
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 4.3.20
+appVersion: 4.3.21

etc/emqx.conf

@@ -1609,14 +1609,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
 ## See: listener.tcp.$name.recbuf
 ##
 ## Value: Bytes
-## listener.ssl.external.recbuf = 4KB
+listener.ssl.external.recbuf = 4KB
 
 ## The TCP send buffer(os kernel) for internal MQTT connections.
 ##
 ## See: listener.tcp.$name.sndbuf
 ##
 ## Value: Bytes
-## listener.ssl.external.sndbuf = 4KB
+listener.ssl.external.sndbuf = 4KB
 
 ## The size of the user-level software buffer used by the driver.
 ##
@@ -1651,6 +1651,12 @@ listener.ssl.external.reuseaddr = true
 ## Value: true | false
 ## listener.ssl.external.gc_after_handshake = false
 
+## hibernate the SSL process after idling for amount of time
+## Default: undefined (off)
+##
+## Value: Duration
+## listener.ssl.external.hibernate_after = 5s
+
 ##--------------------------------------------------------------------
 ## External WebSocket listener for MQTT protocol
 

include/emqx.hrl

@@ -85,9 +85,13 @@
 %%--------------------------------------------------------------------
 
 -record(route, {
+          topic,
+          dest
+         }).
+-type route() :: #route{
                     topic :: binary(),
                     dest  :: node() | {binary(), node()}
-         }).
+                   }.
 
 %%--------------------------------------------------------------------
 %% Plugin
@@ -132,4 +136,3 @@
         }).
 
 -endif.
-

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.20"}).
+-define(EMQX_RELEASE, {opensource, "4.3.21-alpha.1"}).
 
 -else.
 

lib-ce/emqx_dashboard/src/emqx_dashboard.app.src

@@ -1,6 +1,6 @@
 {application, emqx_dashboard,
  [{description, "EMQ X Web Dashboard"},
-  {vsn, "4.3.16"}, % strict semver, bump manually!
+  {vsn, "4.3.17"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_dashboard_sup]},
   {applications, [kernel,stdlib,mnesia,minirest]},

lib-ce/emqx_dashboard/src/emqx_dashboard.erl

@@ -109,6 +109,8 @@ http_handlers() ->
 is_authorized(Req) ->
     is_authorized(binary_to_list(cowboy_req:path(Req)), Req).
 
+is_authorized("/api/v4/emqx_prometheus", _Req) ->
+    true;
 is_authorized("/api/v4/auth", _Req) ->
     true;
 is_authorized(_Path, Req) ->

lib-ce/emqx_modules/src/emqx_mod_delayed.erl

@@ -160,7 +160,7 @@ handle_cast(Msg, State) ->
 
 %% Do Publish...
 handle_info({timeout, TRef, do_publish}, State = #{timer := TRef}) ->
-    DeletedKeys = do_publish(mnesia:dirty_first(?TAB), os:system_time(seconds)),
+    DeletedKeys = do_publish(mnesia:dirty_first(?TAB), erlang:system_time(seconds)),
     lists:foreach(fun(Key) -> mnesia:dirty_delete(?TAB, Key) end, DeletedKeys),
     {noreply, ensure_publish_timer(State#{timer := undefined, publish_at := 0})};
 
@@ -203,11 +203,11 @@ ensure_publish_timer(State) ->
 ensure_publish_timer('$end_of_table', State) ->
     State#{timer := undefined, publish_at := 0};
 ensure_publish_timer({Ts, _Id}, State = #{timer := undefined}) ->
-    ensure_publish_timer(Ts, os:system_time(seconds), State);
+    ensure_publish_timer(Ts, erlang:system_time(seconds), State);
 ensure_publish_timer({Ts, _Id}, State = #{timer := TRef, publish_at := PubAt})
     when Ts < PubAt ->
     ok = emqx_misc:cancel_timer(TRef),
-    ensure_publish_timer(Ts, os:system_time(seconds), State);
+    ensure_publish_timer(Ts, erlang:system_time(seconds), State);
 ensure_publish_timer(_Key, State) ->
     State.
 

lib-ce/emqx_modules/src/emqx_modules.app.src

@@ -1,6 +1,6 @@
 {application, emqx_modules,
  [{description, "EMQ X Module Management"},
-  {vsn, "4.3.9"},
+  {vsn, "4.3.10"},
   {modules, []},
   {applications, [kernel,stdlib]},
   {mod, {emqx_modules_app, []}},

lib-ce/emqx_modules/src/emqx_modules.appup.src

@@ -1,10 +1,14 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.8",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+  [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+   {"4.3.8",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[5-7]">>,
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
-     {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
@@ -31,10 +35,14 @@
      {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.8",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+  [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+   {"4.3.8",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[5-7]">>,
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
-     {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},

lib-ce/emqx_telemetry/src/emqx_telemetry.app.src

@@ -1,6 +1,6 @@
 {application, emqx_telemetry,
  [{description, "EMQ X Telemetry"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_telemetry_sup]},
   {applications, [kernel,stdlib]},

lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src

@@ -1,15 +1,11 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
- [
+  [{"4.3.3",[{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[0-2]">>, [
+   {<<"4\\.3\\.[0-2]">>,
-     {load_module, emqx_telemetry, brutal_purge, soft_purge, []}
+    [{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
-   ]},
+   {<<".*">>,[]}],
-   {<<".*">>, []}
+  [{"4.3.3",[{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
- ],
+   {<<"4\\.3\\.[0-2]">>,
- [
+    [{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[0-2]">>, [
+   {<<".*">>,[]}]}.
-     {load_module, emqx_telemetry, brutal_purge, soft_purge, []}
-   ]},
-   {<<".*">>, []}
- ]
-}.

lib-ce/emqx_telemetry/src/emqx_telemetry.erl

@@ -133,12 +133,6 @@ get_telemetry() ->
 %% gen_server callbacks
 %%--------------------------------------------------------------------
 
-%% This is to suppress dialyzer warnings for mnesia:dirty_write and
-%% dirty_read race condition. Given that the init function is not evaluated
-%% concurrently in one node, it should be free of race condition.
-%% Given the chance of having two nodes bootstraping with the write
-%% is very small, it should be safe to ignore.
--dialyzer([{nowarn_function, [init/1]}]).
 init([Opts]) ->
     State = #state{url = get_value(url, Opts),
                    report_interval = timer:seconds(get_value(report_interval, Opts))},

priv/emqx.schema

@@ -1604,6 +1604,11 @@ end}.
   {datatype, {enum, [true, false]}}
 ]}.
 
+{mapping, "listener.ssl.$name.hibernate_after", "emqx.listeners", [
+  {default, undefined},
+  {datatype, {duration, ms}}
+]}.
+
 %%--------------------------------------------------------------------
 %% MQTT/WebSocket Listeners
 
@@ -2228,7 +2233,8 @@ end}.
                           {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)},
                           {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)},
                           {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)},
-                          {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)}
+                          {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)},
+                          {hibernate_after, cuttlefish:conf_get(Prefix ++ ".hibernate_after", Conf, undefined)}
                          ])
               end,
 

rebar.config

@@ -42,7 +42,7 @@
     , {redbug, "2.0.7"}
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}}
     , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}}
-    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.3"}}}
+    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.4"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}

rebar.config.erl

@@ -103,7 +103,7 @@ plugins(HasElixir) ->
 
 test_plugins() ->
     [ rebar3_proper,
-      {coveralls, {git, "https://github.com/emqx/coveralls-erl", {branch, "fix-git-info"}}}
+      {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
     ].
 
 test_deps() ->

scripts/check-chart-vsn.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")/.."
+
+if [ -f "EMQX_ENTERPRISE" ]; then
+    CHART_FILE='deploy/charts/emqx-ee/Chart.yaml'
+else
+    CHART_FILE='deploy/charts/emqx/Chart.yaml'
+fi
+
+if [ ! -f "$CHART_FILE" ]; then
+    echo "Chart file $CHART_FILE is not found"
+    pwd
+    exit 1
+fi
+
+CHART_VSN="$(grep -oE '^version:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')"
+APP_VSN="$(grep -oE '^appVersion:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')"
+
+if [ "$CHART_VSN" != "$APP_VSN" ]; then
+    echo "Chart version and app version mismatch in $CHART_FILE"
+    exit 2
+fi
+
+PKG_VSN="$(./pkg-vsn.sh | cut -d '-' -f 1)"
+
+if [ "$CHART_VSN" != "$PKG_VSN" ]; then
+    echo "Chart version in $CHART_FILE is not in sync with release version."
+    echo "Chart version: $CHART_VSN"
+    echo "Release version: $PKG_VSN"
+    exit 3
+fi

scripts/macos-sign-binaries.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# intended to run on MacOS only
+# signs all executable files in a given folder (as $1) with developer certificate
+
+# required variables:
+# APPLE_DEVELOPER_IDENTITY: "Developer ID Application: <company name> (<hex id>)"
+# APPLE_DEVELOPER_ID_BUNDLE: base64-encoded content of apple developer id certificate bundle in pksc12 format
+# APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: password used when exporting the bundle
+
+# note: 'bundle' in apple terminology is 'identity'
+
+set -euo pipefail
+
+if [[ "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ]]; then
+    echo "Apple developer certificate is not configured, skip signing"
+    exit 0
+fi
+
+REL_DIR="${1}"
+PKSC12_FILE="$HOME/developer-id-application.p12"
+base64 --decode > "${PKSC12_FILE}" <<<"${APPLE_DEVELOPER_ID_BUNDLE}"
+
+KEYCHAIN='emqx.keychain-db'
+KEYCHAIN_PASSWORD="$(openssl rand -base64 32)"
+
+security create-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security set-keychain-settings -lut 21600 "${KEYCHAIN}"
+security unlock-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security import "${PKSC12_FILE}" -P "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD}" -t cert -f pkcs12 -k "${KEYCHAIN}" -T /usr/bin/codesign
+security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security verify-cert -k "${KEYCHAIN}" -c "${PKSC12_FILE}"
+security find-identity -p codesigning "${KEYCHAIN}"
+
+# add new keychain into the search path for codesign, otherwise the stuff does not work
+keychains=$(security list-keychains -d user)
+keychain_names=();
+for keychain in ${keychains}; do
+    basename=$(basename "${keychain}")
+    keychain_name=${basename::${#basename}-4}
+    keychain_names+=("${keychain_name}")
+done
+security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
+
+# sign
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/asn1-*/priv/lib/asn1rt_nif.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/bcrypt-*/priv/bcrypt_nif.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/crypto-*/priv/lib/{crypto.so,otp_test_engine.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jiffy-*/priv/jiffy.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}

scripts/rel/cut4x.sh

@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+
+## cut a new 4.x release for EMQX (opensource or enterprise).
+
+set -euo pipefail
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
+
+usage() {
+    cat <<EOF
+$0 RELEASE_GIT_TAG [option]
+RELEASE_GIT_TAG is a 'v*' or 'e*' tag for example:
+  v4.3.21
+  e4.4.10-alpha.2
+
+options:
+  -h|--help: Print this usage.
+  -b|--base: Specify the current release base branch, can be one of
+             release-v43, release-v44, release-e43, or release-e44.
+             NOTE: this option should be used when --dryrun.
+  --dryrun:  Do not actually create the git tag.
+  --skip-appup: Skip checking appup
+                Useful when you are sure that appup is already updated'
+
+NOTE: When cutting a 'e*' tag release, both the opensource and enterprise
+      repos should be found as a fetch-remote in the current git repo.
+
+NOTE: For 4.3 series the current working branch must be 'release-v43' for opensource edition
+      and 'release-e43' for enterprise edition.
+      --.--[main-v4.3]------------------.-----------.---
+         \\                             /             \\
+          \`---[release-v43]----(v4.3.X)               \\
+                                       \\               \\
+          .---[release-e43]-------------'--(e4.3.Y)     \\
+         /                                         \\     V
+      --'------[main-v4.3-enterprise]---------------'----'---
+      The same applies to 4.4 series, however, a 4.3 branch is also the upstream branch
+      for the corresponding 4.4 branch. e.g. release-e44 has two upstreams: release-v44 and release-e43
+EOF
+}
+
+logerr() {
+    echo -e "\e[31mERROR: $1\e[39m"
+}
+logmsg() {
+    echo -e "\e[33mINFO: $1\e[39m"
+}
+
+REL_BRANCH_CE="${REL_BRANCH_CE:-release-v43}"
+REL_BRANCH_EE="${REL_BRANCH_EE:-release-e43}"
+
+TAG="${1:-}"
+
+case "$TAG" in
+    v*)
+        if [ -f EMQX_ENTERPRISE ]; then
+            logerr 'Cannot create v-tag on enterprise branch'
+            exit 1
+        fi
+        TAG_PREFIX='v'
+        APPUP_CHECK_PROFILE='emqx'
+        ;;
+    e*)
+        if [ ! -f EMQX_ENTERPRISE ]; then
+            logerr 'Cannot create e-tag on opensource branch'
+            exit 1
+        fi
+        TAG_PREFIX='e'
+        APPUP_CHECK_PROFILE='emqx-ee'
+        ;;
+    -h|--help)
+        usage
+        exit 0
+        ;;
+    *)
+        logerr "Unknown version tag $TAG"
+        usage
+        exit 1
+        ;;
+esac
+
+shift 1
+
+SKIP_APPUP='no'
+DRYRUN='no'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --skip-appup)
+            shift
+            SKIP_APPUP='yes'
+            ;;
+        --dryrun)
+            shift
+            DRYRUN='yes'
+            ;;
+        -b|--base)
+            BASE_BR="${2:-}"
+            if [ -z "${BASE_BR}" ]; then
+                logerr "Must specify which base branch"
+                exit 1
+            fi
+            shift 2
+            ;;
+        *)
+            logerr "Unknown option $1"
+            exit 1
+            ;;
+    esac
+done
+
+rel_branch() {
+    local tag="$1"
+    case "$tag" in
+        v4.3.*)
+            echo 'release-v43'
+            ;;
+        e4.3.*)
+            echo 'release-e43'
+            ;;
+        v4.4.*)
+            echo 'release-v44'
+            ;;
+        e4.4.*)
+            echo 'release-e44'
+            ;;
+        *)
+            logerr "Unsupported version tag $TAG"
+            exit 1
+            ;;
+    esac
+}
+
+## Ensure the current work branch
+assert_work_branch() {
+    local tag="$1"
+    local release_branch
+    release_branch="$(rel_branch "$tag")"
+    local base_branch
+    base_branch="${BASE_BR:-$(git branch --show-current)}"
+    if [ "$base_branch" != "$release_branch" ]; then
+        logerr "Base branch: $base_branch"
+        logerr "Relase tag must be on the release branch: $release_branch"
+        logerr "or must use -b|--base option to specify which release branch is current branch based on"
+        exit 1
+    fi
+}
+assert_work_branch "$TAG"
+
+## Ensure no dirty changes
+assert_not_dirty() {
+    local diff
+    diff="$(git diff --name-only)"
+    if [ -n "$diff" ]; then
+        logerr "Git status is not clean? Changed files:"
+        logerr "$diff"
+        exit 1
+    fi
+}
+assert_not_dirty
+
+## Assert that the tag is not already created
+assert_tag_absent() {
+    local tag="$1"
+    ## Fail if the tag already exists
+    EXISTING="$(git tag --list "$tag")"
+    if [ -n "$EXISTING" ]; then
+        logerr "$tag already released?"
+        logerr 'This script refuse to force re-tag.'
+        logerr 'If re-tag is intended, you must first delete the tag from both local and remote'
+        exit 1
+    fi
+}
+assert_tag_absent "$TAG"
+
+PKG_VSN=$(./pkg-vsn.sh)
+
+## Assert package version is updated to the tag which is being created
+assert_release_version() {
+    local tag="$1"
+    # shellcheck disable=SC2001
+    pkg_vsn="$(echo "$PKG_VSN" | sed 's/-[0-9a-f]\{8\}$//g')"
+    if [ "${TAG_PREFIX}${pkg_vsn}" != "${tag}" ]; then
+        logerr "The release version ($pkg_vsn) is different from the desired git tag."
+        logerr "Update the release version in emqx_release.hrl"
+        exit 1
+    fi
+}
+assert_release_version "$TAG"
+
+## Check if all upstream branches are merged
+if [ -z "${BASE_BR:-}" ]; then
+    ./scripts/rel/sync-remotes.sh
+else
+    ./scripts/rel/sync-remotes.sh --base "$BASE_BR"
+fi
+
+## Check if the Chart versions are in sync
+./scripts/check-chart-vsn.sh
+
+## Check if app versions are bumped
+./scripts/check-apps-vsn.sh
+
+## Ensure appup files are updated
+if [ "$SKIP_APPUP" = 'no' ]; then
+    logmsg "Checking appups"
+    ./scripts/update-appup.sh "$APPUP_CHECK_PROFILE" --check
+else
+    logmsg "Skipped checking appup updates"
+fi
+
+## Ensure relup paths are updated
+case "${PKG_VSN}" in
+    4.3.*)
+        HAS_RELUP_DB='no'
+        ;;
+    *)
+        HAS_RELUP_DB='yes'
+        ;;
+esac
+if [ "$HAS_RELUP_DB" = 'yes' ]; then
+    if [ -f EMQX_ENTERPRISE ]; then
+        RELUP_PATHS='./data/relup-paths-ee.eterm'
+    else
+        RELUP_PATHS='./data/relup-paths.eterm'
+    fi
+    ./scripts/relup-base-vsns.escript check-vsn-db "$PKG_VSN" "$RELUP_PATHS"
+fi
+
+if [ "$DRYRUN" = 'yes' ]; then
+    logmsg "Release tag is ready to be created with command: git tag $TAG"
+else
+    git tag "$TAG"
+    logmsg "$TAG is created OK."
+fi

scripts/rel/sync-remotes.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
+
+CE_BASES=( 'release-v43' 'release-v44' 'main-v4.3' 'main-v4.4' )
+EE_BASES=( 'release-e43' 'release-e44' 'main-v4.3-enterprise' 'main-v4.4-enterprise' )
+
+usage() {
+    cat <<EOF
+$0 [option]
+
+options:
+
+  -h|--help:
+    This script works on one of the branches listed in the -b|--base option below.
+    It tries to merge (by default with --ff-only option)
+    upstreams branches for the current working branch.
+    The uppstream branch of the current branch are as below:
+    * v43: []              # no upstream for 4.3 opensource edition
+    * e43: [v43]           # 4.3 enterprise has 4.3 opensource as upstream
+    * v44: [v43]           # 4.4 opensource has 4.3 opensource as upstream
+    * e44: [e43, v44, v43] # 4.4 enterprise has all the above 3 as upstream
+    For e44:
+      v43 is an indirect upstream.
+      Ideally the merge of v43 should be done through v44 or e43,
+      but it does not hurt to apply a direct merge.
+
+  -b|--base:
+    The base branch of current working branch if currently is not
+    on one of the following branches.
+    ${CE_BASES[@]}
+    ${EE_BASES[@]}
+
+  -i|--interactive:
+    With this option, the script will try to merge upstream
+    branches to local working branch interactively.
+    That is, there will be git prompts to edit commit messages etc.
+    Without this option, the script executes 'git merge' command
+    with '--ff-only' option which conveniently pulls remote
+    updates if there is any, and fails when fast-forward is not possible
+EOF
+}
+
+logerr() {
+    echo -e "\e[31mERROR: $1\e[39m"
+}
+
+logwarn() {
+    echo -e "\e[33mINFO: $1\e[39m"
+}
+
+logmsg() {
+    echo "INFO: $1"
+}
+
+INTERACTIVE='no'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        -i|--interactive)
+            shift
+            INTERACTIVE='yes'
+            ;;
+        -b|--base)
+            shift
+            BASE_BRANCH="$1"
+            shift
+            ;;
+        *)
+            logerr "Unknown option $1"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -f EMQX_ENTERPRISE ]; then
+    IS_ENTERPRISE='yes'
+    BASE_BRANCHES=( "${EE_BASES[@]}" )
+else
+    IS_ENTERPRISE='no'
+    BASE_BRANCHES=( "${CE_BASES[@]}" )
+fi
+
+CURRENT_BRANCH="$(git branch --show-current)"
+BASE_BRANCH="${BASE_BRANCH:-${CURRENT_BRANCH}}"
+
+## check if arg1 is one of the elements in arg2-N
+is_element() {
+    local e match="$1"
+    shift
+    for e in "${@}"; do
+        if [ "$e" = "$match" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+if ! is_element "$BASE_BRANCH" "${BASE_BRANCHES[@]}"; then
+    logerr "Cannot work with branch $BASE_BRANCH"
+    logerr "The base branch must be one of: ${BASE_BRANCHES[*]}"
+    logerr "Change work branch to one of the above."
+    logerr "OR: use -b|--base to specify from which base branch is current working branch created"
+    exit 1
+fi
+
+## Find git remotes to fetch from.
+##
+## NOTE: For enterprise, the opensource repo must be added as a remote.
+##       Because not all changes in opensource repo are synced to enterprise repo immediately.
+##
+## NOTE: grep -v enterprise here, but why not to match on full repo name 'emqx/emqx.git'?
+##       It's because the git remote does not always end with .git
+GIT_REMOTE_CE="$(git remote -v | grep 'emqx/emqx' | grep -v enterprise | grep fetch | head -1 | awk '{print $1}' || true)"
+if [ -z "$GIT_REMOTE_CE" ]; then
+	logerr "Cannot find git remote for emqx/emqx"
+    exit 1
+fi
+REMOTES=( "${GIT_REMOTE_CE}" )
+if [ "$IS_ENTERPRISE" = 'yes' ]; then
+    GIT_REMOTE_EE="$(git remote -v | grep 'emqx/emqx-enterprise' | grep fetch | head -1 | awk '{print $1}' || true)"
+    if [ -z "$GIT_REMOTE_EE" ]; then
+        logerr "Cannot find git remote for emqx/emqx-enterprise"
+        exit 1
+    fi
+    REMOTES+=( "$GIT_REMOTE_EE" )
+fi
+
+## Fetch the remotes
+for remote in "${REMOTES[@]}"; do
+    logwarn "Fetching from remote=${remote} (force tag sync)."
+    git fetch "$remote" --tags --force
+done
+
+logmsg 'Fetched all remotes'
+
+if [ "$INTERACTIVE" = 'yes' ]; then
+    MERGE_OPTS=''
+else
+    ## Using --ff-only to *check* if the remote is already merged
+    ## Also conveniently merged it in case it's *not* merged but can be fast-forwarded
+    ## Alternative is to check with 'git merge-base'
+    MERGE_OPTS='--ff-only'
+fi
+
+## Get the git remote reference of the given 'release-' or 'main-' branch
+remote_ref() {
+    local branch="$1"
+    if is_element "$branch" "${EE_BASES[@]}"; then
+        echo -n "${GIT_REMOTE_EE}/${branch} "
+    else
+        echo -n "${GIT_REMOTE_CE}/${branch} "
+    fi
+}
+
+remote_refs() {
+    local br
+    for br in "${@}"; do
+        remote_ref "$br"
+    done
+}
+
+## Get upstream branches of the given branch
+upstream_branches() {
+    local base="$1"
+    case "$base" in
+        release-v43|main-v4.3)
+            ## no upstream for 4.3 opensource
+            remote_ref "$base"
+            ;;
+        release-v44)
+            remote_refs "$base" 'release-v43'
+            ;;
+        main-v4.4)
+            remote_refs "$base" 'main-v4.3'
+            ;;
+        release-e43)
+            remote_refs "$base" 'release-v43'
+            ;;
+        main-v4.3-enterprise)
+            remote_refs "$base" 'main-v4.3'
+            ;;
+        release-e44)
+            remote_refs "$base" 'release-v44' 'release-e43' 'release-v43'
+            ;;
+        main-v4.4-enterprise)
+            remote_refs "$base" 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3'
+            ;;
+    esac
+}
+
+for remote_ref in $(upstream_branches "$BASE_BRANCH"); do
+    logmsg "Merging $remote_ref"
+    git merge $MERGE_OPTS "$remote_ref"
+done

scripts/update_appup.escript

@@ -3,6 +3,9 @@
 
 -mode(compile).
 
+-define(RED, "\e[31m").
+-define(RESET, "\e[39m").
+
 usage() ->
 "A script that fills in boilerplate for appup files.
 
@@ -59,7 +62,8 @@ app_specific_actions(_) ->
 ignored_apps() ->
     [gpb, %% only a build tool
      emqx_dashboard, %% generic appup file for all versions
-     emqx_management %% generic appup file for all versions
+     emqx_management, %% generic appup file for all versions
+     emqx_modules_spec %% generic appup file for all versions
     ] ++ otp_standard_apps().
 
 main(Args) ->
@@ -109,7 +113,7 @@ changes, supervisor changes, process restarts and so on. Also the load order of
 the beam files might need updating.~n"),
     halt(0);
 warn_and_exit(false) ->
-    log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"),
+    logerr("Incomplete appups found. Please inspect the output for more details.~n", []),
     halt(1).
 
 prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) ->
@@ -281,9 +285,9 @@ merge_update_actions(App, Changes, Vsns, PrevVersion) ->
 %% but there is a 1.1.2 in appup we may skip merging instructions for
 %% 1.1.2 because it's not used and no way to know what has been changed
 is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) ->
-    case is_app_external(App) andalso parse_version_number(Vsn) of
+    case is_app_external(App) andalso parse_version(Vsn, non_strict_semver) of
         {ok, VsnTuple} ->
-            case parse_version_number(PrevVersion) of
+            case parse_version(PrevVersion, non_strict_semver) of
                 {ok, PrevVsnTuple} ->
                     VsnTuple > PrevVsnTuple;
                 _ ->
@@ -309,7 +313,7 @@ do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) ->
             true ->
                 [];
             false ->
-                [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed] ++
+                [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed, not is_secret_module(M)] ++
                 [{add_module, M} || M <- New]
         end,
     {OldActionsWithStop, OldActionsAfterStop} =
@@ -321,10 +325,18 @@ do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) ->
             true ->
                 [];
             false ->
-                [{delete_module, M} || M <- Deleted]
+                [{delete_module, M} || M <- Deleted, not is_secret_module(M)]
         end ++
         AppSpecific.
 
+%% Do not reload or delet _secret modules
+is_secret_module(Module) ->
+    Suffix = "_secret",
+    case string:right(atom_to_list(Module), length(Suffix)) of
+        Suffix -> true;
+        _ -> false
+    end.
+
 %% If an entry restarts an application, there's no need to use
 %% `load_module' instructions.
 contains_restart_application(Application, Actions) ->
@@ -394,7 +406,7 @@ contains_version(Needle, Haystack) when is_list(Needle) ->
 %% past versions that should be covered by regexes in .appup file
 %% instructions.
 enumerate_past_versions(Vsn) when is_list(Vsn) ->
-    case parse_version_number(Vsn) of
+    case parse_version(Vsn) of
         {ok, ParsedVsn} ->
             {ok, enumerate_past_versions(ParsedVsn)};
         Error ->
@@ -403,14 +415,39 @@ enumerate_past_versions(Vsn) when is_list(Vsn) ->
 enumerate_past_versions({Major, Minor, Patch}) ->
     [{Major, Minor, P} || P <- lists:seq(Patch - 1, 0, -1)].
 
-parse_version_number(Vsn) when is_list(Vsn) ->
+parse_version(Vsn) ->
-    Nums = string:split(Vsn, ".", all),
+    parse_version(Vsn, strict_semver).
-    Results = lists:map(fun string:to_integer/1, Nums),
+
-    case Results of
+parse_version(Vsn, MaybeSemver) when is_list(Vsn) ->
-        [{Major, []}, {Minor, []}, {Patch, []}] ->
+    case parse_dot_separated_numbers(Vsn) of
-            {ok, {Major, Minor, Patch}};
+        {ok, {_Major, _Minor, _Patch}} = Res ->
+            Res;
+        {ok, Nums} ->
+            case MaybeSemver of
+                strict_semver ->
+                    {error, {bad_semver, Vsn}};
+                non_strict_semver ->
+                    {ok, Nums}
+            end;
+        {error, Reason} ->
+            {error, {Reason, Vsn}}
+    end.
+
+parse_dot_separated_numbers(Str) when is_list(Str) ->
+    try
+        Split = string:split(Str, ".", all),
+        IntL = lists:map(fun(SubStr) ->
+                                 case string:to_integer(SubStr) of
+                                     {Int, []} when is_integer(Int) ->
+                                         Int;
                                      _ ->
-            {error, bad_version}
+                                         throw(no_integer)
+                                 end
+                         end, Split),
+        {ok, list_to_tuple(IntL)}
+    catch
+        _ : _ ->
+            {error, bad_version_string}
     end.
 
 vsn_number_to_string({Major, Minor, Patch}) ->
@@ -474,7 +511,7 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
             ok;
         {diffs, Diffs} ->
             set_invalid(),
-            log("ERROR: Appup file for '~p' is not complete.~n"
+            logerr("Appup file for '~p' is not complete.~n"
                    "Missing:~100p~n", [App, Diffs]),
             notok
     end.
@@ -494,7 +531,7 @@ render_appup(App, File, Up, Down) ->
             do_render_appup(File, Up, Down);
         {error, enoent} when IsCheck ->
             %% failed to read old file, exit
-            log("ERROR: ~s is missing", [File]),
+            logerr("~s is missing", [File]),
             set_invalid()
     end.
 
@@ -580,7 +617,7 @@ diff_app(UpOrDown, App,
             case UpOrDown =:= up of
                 true ->
                     %% only log for the upgrade case because it would be the same result
-                    log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
+                    logerr("Application '~p' contains changes, but its version is not updated. ~s",
                            [App, format_changes(Changes)]);
                 false ->
                     ok
@@ -723,5 +760,8 @@ log(Msg) ->
 log(Msg, Args) ->
     io:format(standard_error, Msg, Args).
 
+logerr(Msg, Args) ->
+    io:format(standard_error,  ?RED ++ "ERROR: "++ Msg ++ ?RESET, Args).
+
 otp_standard_apps() ->
     [ssl, mnesia, kernel, asn1, stdlib].

scripts/xref-check.escript

@@ -0,0 +1,58 @@
+#!/usr/bin/env escript
+
+%%% @doc
+%%% Unlike rebar3 xref check, this script runs the full xref checks in the EMQX release dir,
+%%% meaning all the modules for release are analysed.
+%%% For behavior configuration, all rebar3 related modules attributes, filters are not used in this script,
+%%% instead all the filters, checks are defined in `xref_check.eterm`
+main(_) ->
+    {ok, [Jobs]} = file:consult("scripts/xref_check.eterm"),
+    lists:foreach(fun(J) -> do_check(J) end, Jobs),
+    case get(is_warn_found) of
+        true ->
+            halt(1);
+        _ ->
+            ok
+    end.
+
+do_check(#{ name := Name
+          , analysis := Analysis
+          , excl_apps := ExclApps
+          , excl_mods := ExclMods
+          , filters := Filters
+          }) ->
+    xref:start(Name),
+    %% Build a table for later printing more informative warnings.
+    %% The table is currently not in use.
+    Tid = ets:new(Name, [ordered_set, named_table]),
+    xref:set_default(Name, [{verbose,false}, {warnings,false}]),
+    Profile = case filelib:is_file("EMQX_ENTERPRISE") of
+                  true -> 'emqx-ee';
+                  false -> emqx
+              end,
+    Dir = filename:join(["_build/",  Profile, "rel/emqx/lib/"]),
+    xref:add_release(Name, Dir),
+    xref:add_application(Name, code:lib_dir(erts)),
+    [ case xref:remove_application(Name, App) of
+          ok -> ok;
+          {error, xref_base, {no_such_application, _}} -> ok
+      end || App <- ExclApps
+    ],
+
+    [case xref:remove_module(Name, M) of
+         ok -> ok;
+         %% but in doc it should return '{error, module(), Reason}`
+         {error, xref_base, {no_such_module, M}} -> ok
+     end || M <- ExclMods
+    ],
+    ModuleInfos = xref:info(Name, modules),
+    LibInfos = xref:info(Name, libraries),
+    true = ets:insert(Tid, ModuleInfos ++ LibInfos),
+    {ok, Res0} = xref:analyse(Name, Analysis),
+    Res = Res0 -- Filters,
+    Res =/= [] andalso
+        begin
+            put(is_warn_found, true),
+            io:format("** Warnings for ~p~n : ~p~n", [Name, Res])
+        end,
+    xref:stop(Name).

scripts/xref_check.eterm

@@ -0,0 +1,537 @@
+%% -*- mode: erlang; -*-
+[ %% Check undefined_function_calls
+  #{ name => undefined_function_calls
+   , analysis => undefined_function_calls
+   , excl_apps =>
+         [ observer
+         , redbug
+         ]
+   , excl_mods =>
+         [ hipe_unified_loader
+         , systools_make
+         , systools
+         , systools_relup
+         , basho_bench_driver_erldis
+         , release_handler
+         , cuttlefish_rebar_plugin
+         ]
+   , filters =>
+         [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}},
+          {{coap_client,channel_apply,3},{coap_dtls_socket,connect,2}},
+          {{coap_client,channel_apply,3},{coap_udp_socket,close,1}},
+          {{coap_client,channel_apply,3},{coap_udp_socket,get_channel,2}},
+          {{coap_client,channel_apply,3},{coap_udp_socket,start_link,0}},
+          {{coap_client,test,0},{eunit,test,1}},
+          {{coap_core_link,test,0},{eunit,test,1}},
+          {{coap_message_parser,test,0},{eunit,test,1}},
+          {{coap_observer,init,1},{coap_dtls_socket,connect,2}},
+          {{coap_observer,init,1},{coap_udp_socket,get_channel,2}},
+          {{coap_observer,init,1},{coap_udp_socket,start_link,0}},
+          {{coap_observer,terminate,2},{coap_dtls_socket,close,1}},
+          {{coap_observer,terminate,2},{coap_udp_socket,close,1}},
+
+          %% Exclude hipe which is disabled
+          {{code,beam_file_native_md5,2},{hipe_unified_loader,chunk_name,1}},
+          {{code,get_native_fun,0},{hipe_unified_loader,chunk_name,1}},
+          {{code,load_native_code_for_all_loaded,1},
+           {hipe_unified_loader,chunk_name,1}},
+          {{code_server,handle_call,3},{hipe_unified_loader,load,3}},
+          {{code_server,handle_call,3},{hipe_unified_loader,load_module,4}},
+          {{code_server,try_load_module_2,6},
+           {hipe_unified_loader,load_native_code,3}},
+          {{compile,embed_native_code,2},{hipe_unified_loader,chunk_name,1}},
+          {{compile,native_compile_1,2},{hipe,compile,4}},
+
+
+          {{core_link,test,0},{eunit,test,1}},
+          {{emqx_misc,ipv6_probe,1},{gen_tcp,ipv6_probe,0}},
+          {{eredis_parser,test,0},{eunit,test,1}},
+          {{gen_rpc_driver_ssl,authenticate_client,3},
+           {ssl_verify_hostname,verify_cert_hostname,2}},
+          {{gen_rpc_driver_ssl,merge_ssl_options,2},
+           {ssl_verify_hostname,verify_fun,3}},
+          {{gproc_dist,from_leader,3},{gen_leader,leader_cast,2}},
+          {{gproc_dist,handle_call,4},{gen_leader,leader_node,1}},
+          {{gproc_dist,handle_leader_cast,3},{gen_leader,alive,1}},
+          {{gproc_dist,handle_leader_cast,3},{gen_leader,broadcast,3}},
+          {{gproc_dist,initiate_sync,3},{gen_leader,alive,1}},
+          {{gproc_dist,initiate_sync,3},{gen_leader,broadcast,3}},
+          {{gproc_dist,leader_call,1},{gen_leader,leader_call,2}},
+          {{gproc_dist,leader_cast,1},{gen_leader,leader_cast,2}},
+          {{gproc_dist,send_sync_complete,3},{gen_leader,broadcast,3}},
+          {{gproc_dist,start_link,1},{gen_leader,start_link,6}},
+          {{inets_ftp_wrapper,start_service,1},{ftp,start_service,1}},
+          {{inets_ftp_wrapper,start_standalone,1},{ftp,start_standalone,1}},
+          {{inets_ftp_wrapper,stop_service,1},{ftp,stop_service,1}},
+          {{inets_tftp_wrapper,start_service,1},{tftp,start_service,1}},
+          {{inets_tftp_wrapper,start_standalone,1},{tftp,start_standalone,1}},
+          {{inets_tftp_wrapper,stop_service,1},{tftp,stop_service,1}},
+          {{jose_chacha20_poly1305_libsodium,authenticate,3},
+           {libsodium_crypto_onetimeauth_poly1305,crypto_onetimeauth_poly1305,2}},
+          {{jose_chacha20_poly1305_libsodium,decrypt,5},
+           {libsodium_crypto_aead_chacha20poly1305,ietf_decrypt_detached,5}},
+          {{jose_chacha20_poly1305_libsodium,encrypt,4},
+           {libsodium_crypto_aead_chacha20poly1305,ietf_encrypt_detached,4}},
+          {{jose_chacha20_poly1305_libsodium,one_time_key,2},
+           {libsodium_crypto_stream_chacha20,ietf_xor_ic,4}},
+          {{jose_chacha20_poly1305_libsodium,verify,4},
+           {libsodium_crypto_onetimeauth_poly1305,verify,3}},
+          {{jose_curve25519_libdecaf,ed25519_sign,2},
+           {libdecaf_curve25519,ed25519_sign,2}},
+          {{jose_curve25519_libdecaf,ed25519_verify,3},
+           {libdecaf_curve25519,ed25519_verify,3}},
+          {{jose_curve25519_libdecaf,ed25519ph_sign,2},
+           {libdecaf_curve25519,ed25519ph_sign,2}},
+          {{jose_curve25519_libdecaf,ed25519ph_verify,3},
+           {libdecaf_curve25519,ed25519ph_verify,3}},
+          {{jose_curve25519_libdecaf,eddsa_keypair,0},
+           {libdecaf_curve25519,eddsa_keypair,0}},
+          {{jose_curve25519_libdecaf,eddsa_keypair,1},
+           {libdecaf_curve25519,eddsa_keypair,1}},
+          {{jose_curve25519_libdecaf,eddsa_secret_to_public,1},
+           {libdecaf_curve25519,eddsa_secret_to_pk,1}},
+          {{jose_curve25519_libdecaf,x25519_keypair,0},
+           {libdecaf_curve25519,x25519_keypair,0}},
+          {{jose_curve25519_libdecaf,x25519_keypair,1},
+           {libdecaf_curve25519,x25519_keypair,1}},
+          {{jose_curve25519_libdecaf,x25519_secret_to_public,1},
+           {libdecaf_curve25519,x25519,1}},
+          {{jose_curve25519_libdecaf,x25519_shared_secret,2},
+           {libdecaf_curve25519,x25519,2}},
+          {{jose_curve25519_libsodium,ed25519_sign,2},
+           {libsodium_crypto_sign_ed25519,detached,2}},
+          {{jose_curve25519_libsodium,ed25519_verify,3},
+           {libsodium_crypto_sign_ed25519,verify_detached,3}},
+          {{jose_curve25519_libsodium,ed25519ph_sign,2},
+           {libsodium_crypto_hash_sha512,crypto_hash_sha512,1}},
+          {{jose_curve25519_libsodium,ed25519ph_verify,3},
+           {libsodium_crypto_hash_sha512,crypto_hash_sha512,1}},
+          {{jose_curve25519_libsodium,eddsa_keypair,0},
+           {libsodium_crypto_sign_ed25519,keypair,0}},
+          {{jose_curve25519_libsodium,eddsa_keypair,1},
+           {libsodium_crypto_sign_ed25519,seed_keypair,1}},
+          {{jose_curve25519_libsodium,eddsa_secret_to_public,1},
+           {libsodium_crypto_sign_ed25519,seed_keypair,1}},
+          {{jose_curve25519_libsodium,x25519_keypair,0},
+           {libsodium_crypto_box_curve25519xsalsa20poly1305,keypair,0}},
+          {{jose_curve25519_libsodium,x25519_secret_to_public,1},
+           {libsodium_crypto_scalarmult_curve25519,base,1}},
+          {{jose_curve25519_libsodium,x25519_shared_secret,2},
+           {libsodium_crypto_scalarmult_curve25519,crypto_scalarmult_curve25519,2}},
+          {{jose_curve448_libdecaf,ed448_sign,2},{libdecaf_curve448,ed448_sign,2}},
+          {{jose_curve448_libdecaf,ed448_sign,3},{libdecaf_curve448,ed448_sign,3}},
+          {{jose_curve448_libdecaf,ed448_verify,3},
+           {libdecaf_curve448,ed448_verify,3}},
+          {{jose_curve448_libdecaf,ed448_verify,4},
+           {libdecaf_curve448,ed448_verify,4}},
+          {{jose_curve448_libdecaf,ed448ph_sign,2},
+           {libdecaf_curve448,ed448ph_sign,2}},
+          {{jose_curve448_libdecaf,ed448ph_sign,3},
+           {libdecaf_curve448,ed448ph_sign,3}},
+          {{jose_curve448_libdecaf,ed448ph_verify,3},
+           {libdecaf_curve448,ed448ph_verify,3}},
+          {{jose_curve448_libdecaf,ed448ph_verify,4},
+           {libdecaf_curve448,ed448ph_verify,4}},
+          {{jose_curve448_libdecaf,eddsa_keypair,0},
+           {libdecaf_curve448,eddsa_keypair,0}},
+          {{jose_curve448_libdecaf,eddsa_keypair,1},
+           {libdecaf_curve448,eddsa_keypair,1}},
+          {{jose_curve448_libdecaf,eddsa_secret_to_public,1},
+           {libdecaf_curve448,eddsa_secret_to_pk,1}},
+          {{jose_curve448_libdecaf,x448_keypair,0},
+           {libdecaf_curve448,x448_keypair,0}},
+          {{jose_curve448_libdecaf,x448_keypair,1},
+           {libdecaf_curve448,x448_keypair,1}},
+          {{jose_curve448_libdecaf,x448_secret_to_public,1},
+           {libdecaf_curve448,x448,1}},
+          {{jose_curve448_libdecaf,x448_shared_secret,2},{libdecaf_curve448,x448,2}},
+          {{jose_json_jason,decode,1},{'Elixir.Jason','decode!',1}},
+          {{jose_json_jason,encode,1},{'Elixir.Jason','encode!',1}},
+          {{jose_json_jsone,decode,1},{jsone,decode,1}},
+          {{jose_json_jsone,encode,1},{jsone,encode,2}},
+          {{jose_json_jsx,decode,1},{jsx,decode,2}},
+          {{jose_json_jsx,encode,1},{jsx,encode,1}},
+          {{jose_json_ojson,decode,1},{ojson,'decode!',1}},
+          {{jose_json_ojson,encode,1},{ojson,'encode!',1}},
+          {{jose_json_poison,decode,1},{'Elixir.Poison','decode!',1}},
+          {{jose_json_poison,encode,1},{'Elixir.Poison','encode!',1}},
+          {{jose_json_poison_compat_encoder,decode,1},{'Elixir.Poison','decode!',1}},
+          {{jose_json_poison_compat_encoder,encode,1},
+           {'Elixir.IO',iodata_to_binary,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode,1},
+           {'Elixir.Poison.EncodeError',exception,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode,1},
+           {'Elixir.Poison.Encoder.Atom',encode,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode,1},
+           {'Elixir.Poison.Encoder.BitString',encode,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode,1},
+           {'Elixir.Poison.Encoder.Float',encode,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode,1},
+           {'Elixir.Poison.Encoder.Integer',encode,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode_map,1},
+           {'Elixir.Poison.Encoder.BitString',encode,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode_name,1},
+           {'Elixir.Atom',to_string,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode_name,1},
+           {'Elixir.Kernel',inspect,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode_name,1},
+           {'Elixir.Poison.EncodeError',exception,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode_struct,2},
+           {'Elixir.Enum',flat_map,2}},
+          {{jose_json_poison_compat_encoder,lexical_encode_struct,2},
+           {'Elixir.HashDict',size,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode_struct,2},
+           {'Elixir.Map',from_struct,1}},
+          {{jose_json_poison_compat_encoder,lexical_encode_struct,2},
+           {'Elixir.Poison.Encoder.BitString',encode,2}},
+          {{jose_json_poison_lexical_encoder,decode,1},
+           {'Elixir.Poison','decode!',1}},
+          {{jose_json_poison_lexical_encoder,encode,1},
+           {'Elixir.JOSE.Poison','lexical_encode!',1}},
+          {{jose_jwk_kty_rsa,try_generate_key,3},{cutkey,rsa,3}},
+          {{jose_sha3_keccakf1600_driver,sha3_224,1},
+           {keccakf1600_fips202,sha3_224,1}},
+          {{jose_sha3_keccakf1600_driver,sha3_256,1},
+           {keccakf1600_fips202,sha3_256,1}},
+          {{jose_sha3_keccakf1600_driver,sha3_384,1},
+           {keccakf1600_fips202,sha3_384,1}},
+          {{jose_sha3_keccakf1600_driver,sha3_512,1},
+           {keccakf1600_fips202,sha3_512,1}},
+          {{jose_sha3_keccakf1600_driver,shake128,2},
+           {keccakf1600_fips202,shake128,2}},
+          {{jose_sha3_keccakf1600_driver,shake256,2},
+           {keccakf1600_fips202,shake256,2}},
+          {{jose_sha3_keccakf1600_nif,sha3_224,1},{keccakf1600,hash,2}},
+          {{jose_sha3_keccakf1600_nif,sha3_256,1},{keccakf1600,hash,2}},
+          {{jose_sha3_keccakf1600_nif,sha3_384,1},{keccakf1600,hash,2}},
+          {{jose_sha3_keccakf1600_nif,sha3_512,1},{keccakf1600,hash,2}},
+          {{jose_sha3_keccakf1600_nif,shake128,2},{keccakf1600,hash,3}},
+          {{jose_sha3_keccakf1600_nif,shake256,2},{keccakf1600,hash,3}},
+          {{jose_sha3_libdecaf,sha3_224,1},{libdecaf_sha3,hash,2}},
+          {{jose_sha3_libdecaf,sha3_256,1},{libdecaf_sha3,hash,2}},
+          {{jose_sha3_libdecaf,sha3_384,1},{libdecaf_sha3,hash,2}},
+          {{jose_sha3_libdecaf,sha3_512,1},{libdecaf_sha3,hash,2}},
+          {{jose_sha3_libdecaf,shake128,2},{libdecaf_sha3,hash,3}},
+          {{jose_sha3_libdecaf,shake256,2},{libdecaf_sha3,hash,3}},
+          {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_dtls_socket,close,1}},
+          {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_dtls_socket,connect,2}},
+          {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_udp_socket,close,1}},
+          {{lwm2m_coap_client,channel_apply,3},
+           {lwm2m_coap_udp_socket,get_channel,2}},
+          {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_udp_socket,start_link,0}},
+          {{lwm2m_coap_client,test,0},{eunit,test,1}},
+          {{lwm2m_coap_message_parser,test,0},{eunit,test,1}},
+          {{lwm2m_coap_observer,init,1},{lwm2m_coap_dtls_socket,connect,2}},
+          {{lwm2m_coap_observer,init,1},{lwm2m_coap_udp_socket,get_channel,2}},
+          {{lwm2m_coap_observer,init,1},{lwm2m_coap_udp_socket,start_link,0}},
+          {{lwm2m_coap_observer,terminate,2},{lwm2m_coap_dtls_socket,close,1}},
+          {{lwm2m_coap_observer,terminate,2},{lwm2m_coap_udp_socket,close,1}},
+          {{ranch_app,consider_profiling,0},{eprof,start,0}},
+          {{ranch_app,consider_profiling,0},{eprof,start_profiling,1}},
+          {{ranch_app,profile_output,0},{eprof,analyze,1}},
+          {{ranch_app,profile_output,0},{eprof,log,1}},
+          {{ranch_app,profile_output,0},{eprof,stop_profiling,0}}
+         ]
+   }
+
+  %% Check undefined_functions
+, #{ name => undefined_functions
+   , analysis => undefined_functions
+   , excl_apps =>
+         [ observer
+         ]
+   , excl_mods =>
+         [ systools
+         , systools_make
+         , release_handler
+         , systools_relup
+         , cuttlefish_rebar_plugin
+         ]
+   , filters =>
+         [{'Elixir.Atom',to_string,1},
+          {'Elixir.Enum',flat_map,2},
+          {'Elixir.HashDict',size,1},
+          {'Elixir.IO',iodata_to_binary,1},
+          {'Elixir.JOSE.Poison','lexical_encode!',1},
+          {'Elixir.Jason','decode!',1},
+          {'Elixir.Jason','encode!',1},
+          {'Elixir.Kernel',inspect,1},
+          {'Elixir.Map',from_struct,1},
+          {'Elixir.Poison','decode!',1},
+          {'Elixir.Poison','encode!',1},
+          {'Elixir.Poison.EncodeError',exception,1},
+          {'Elixir.Poison.Encoder.Atom',encode,2},
+          {'Elixir.Poison.Encoder.BitString',encode,2},
+          {'Elixir.Poison.Encoder.Float',encode,2},
+          {'Elixir.Poison.Encoder.Integer',encode,2},
+          {coap_dtls_socket,close,1},
+          {coap_dtls_socket,connect,2},
+          {coap_udp_socket,close,1},
+          {coap_udp_socket,get_channel,2},
+          {coap_udp_socket,start_link,0},
+          {ct_slave,start,2},
+          {ct_slave,stop,1},
+          {cutkey,rsa,3},
+          {eprof,analyze,1},
+          {eprof,log,1},
+          {eprof,start,0},
+          {eprof,start_profiling,1},
+          {eprof,stop_profiling,0},
+          {erldis,mget,2},
+          {erldis,set,3},
+          {erldis_client,connect,0},
+          {et_collector,iterate,5},
+          {et_collector,report,2},
+          {et_selector,parse_event,2},
+          {et_viewer,get_collector_pid,1},
+          {et_viewer,start_link,1},
+          {eunit,test,1},
+          {ftp,start_service,1},
+          {ftp,start_standalone,1},
+          {ftp,stop_service,1},
+          {gen_leader,alive,1},
+          {gen_leader,broadcast,3},
+          {gen_leader,leader_call,2},
+          {gen_leader,leader_cast,2},
+          {gen_leader,leader_node,1},
+          {gen_leader,start_link,6},
+          {gen_tcp,ipv6_probe,0},
+
+          %% We dont use hipes
+          {hipe,compile,4},
+          {hipe_bifs,add_ref,2},
+          {hipe_bifs,alloc_data,3},
+          {hipe_bifs,alloc_loader_state,1},
+          {hipe_bifs,atom_to_word,1},
+          {hipe_bifs,bif_address,3},
+          {hipe_bifs,check_crc,1},
+          {hipe_bifs,commit_patch_load,1},
+          {hipe_bifs,enter_code,3},
+          {hipe_bifs,enter_sdesc,2},
+          {hipe_bifs,find_na_or_make_stub,1},
+          {hipe_bifs,fun_to_address,1},
+          {hipe_bifs,get_fe,2},
+          {hipe_bifs,merge_term,1},
+          {hipe_bifs,patch_call,3},
+          {hipe_bifs,patch_insn,3},
+          {hipe_bifs,primop_address,1},
+          {hipe_bifs,set_funinfo_native_address,3},
+          {hipe_bifs,set_native_address,3},
+          {hipe_bifs,set_native_address_in_fe,2},
+          {hipe_bifs,term_to_word,1},
+          {hipe_bifs,write_u32,2},
+          {hipe_bifs,write_u64,2},
+          {hipe_bifs,write_u8,2},
+
+
+          {jsone,decode,1},
+          {jsone,encode,2},
+          {jsx,decode,2},
+          {jsx,encode,1},
+          {keccakf1600,hash,2},
+          {keccakf1600,hash,3},
+          {keccakf1600_fips202,sha3_224,1},
+          {keccakf1600_fips202,sha3_256,1},
+          {keccakf1600_fips202,sha3_384,1},
+          {keccakf1600_fips202,sha3_512,1},
+          {keccakf1600_fips202,shake128,2},
+          {keccakf1600_fips202,shake256,2},
+          {libdecaf_curve25519,ed25519_sign,2},
+          {libdecaf_curve25519,ed25519_verify,3},
+          {libdecaf_curve25519,ed25519ph_sign,2},
+          {libdecaf_curve25519,ed25519ph_verify,3},
+          {libdecaf_curve25519,eddsa_keypair,0},
+          {libdecaf_curve25519,eddsa_keypair,1},
+          {libdecaf_curve25519,eddsa_secret_to_pk,1},
+          {libdecaf_curve25519,x25519,1},
+          {libdecaf_curve25519,x25519,2},
+          {libdecaf_curve25519,x25519_keypair,0},
+          {libdecaf_curve25519,x25519_keypair,1},
+          {libdecaf_curve448,ed448_sign,2},
+          {libdecaf_curve448,ed448_sign,3},
+          {libdecaf_curve448,ed448_verify,3},
+          {libdecaf_curve448,ed448_verify,4},
+          {libdecaf_curve448,ed448ph_sign,2},
+          {libdecaf_curve448,ed448ph_sign,3},
+          {libdecaf_curve448,ed448ph_verify,3},
+          {libdecaf_curve448,ed448ph_verify,4},
+          {libdecaf_curve448,eddsa_keypair,0},
+          {libdecaf_curve448,eddsa_keypair,1},
+          {libdecaf_curve448,eddsa_secret_to_pk,1},
+          {libdecaf_curve448,x448,1},
+          {libdecaf_curve448,x448,2},
+          {libdecaf_curve448,x448_keypair,0},
+          {libdecaf_curve448,x448_keypair,1},
+          {libdecaf_sha3,hash,2},
+          {libdecaf_sha3,hash,3},
+          {libsodium_crypto_aead_chacha20poly1305,ietf_decrypt_detached,5},
+          {libsodium_crypto_aead_chacha20poly1305,ietf_encrypt_detached,4},
+          {libsodium_crypto_box_curve25519xsalsa20poly1305,keypair,0},
+          {libsodium_crypto_hash_sha512,crypto_hash_sha512,1},
+          {libsodium_crypto_onetimeauth_poly1305,crypto_onetimeauth_poly1305,2},
+          {libsodium_crypto_onetimeauth_poly1305,verify,3},
+          {libsodium_crypto_scalarmult_curve25519,base,1},
+          {libsodium_crypto_scalarmult_curve25519,crypto_scalarmult_curve25519,2},
+          {libsodium_crypto_sign_ed25519,detached,2},
+          {libsodium_crypto_sign_ed25519,keypair,0},
+          {libsodium_crypto_sign_ed25519,seed_keypair,1},
+          {libsodium_crypto_sign_ed25519,verify_detached,3},
+          {libsodium_crypto_stream_chacha20,ietf_xor_ic,4},
+          {lwm2m_coap_dtls_socket,close,1},
+          {lwm2m_coap_dtls_socket,connect,2},
+          {lwm2m_coap_udp_socket,close,1},
+          {lwm2m_coap_udp_socket,get_channel,2},
+          {lwm2m_coap_udp_socket,start_link,0},
+          {ojson,'decode!',1},
+          {ojson,'encode!',1},
+          {ssl_verify_hostname,verify_cert_hostname,2},
+          {ssl_verify_hostname,verify_fun,3},
+          {tftp,start_service,1},
+          {tftp,start_standalone,1},
+          {tftp,stop_service,1},
+
+          %% Xref could not check itself
+          {xref,add_application,3},
+          {xref,analyze,2},
+          {xref,set_default,3},
+          {xref,set_library_path,2},
+          {xref,start,2},
+          {xref,stop,1}
+         ]
+   }
+
+, #{ name => locals_not_used
+   , analysis => locals_not_used
+   , excl_apps =>
+         [
+         ]
+   , excl_mods =>
+         [
+         ]
+   , filters =>
+         [{coap_core_link_parser,return_error,2},
+          {core_link_parser,return_error,2},
+          {emqx_exhook_pb,e_type_double,3},
+          {emqx_exhook_pb,e_type_fixed32,3},
+          {emqx_exhook_pb,e_type_fixed64,3},
+          {emqx_exhook_pb,e_type_float,3},
+          {emqx_exhook_pb,e_type_int32,3},
+          {emqx_exhook_pb,e_type_int64,3},
+          {emqx_exhook_pb,e_type_sfixed32,3},
+          {emqx_exhook_pb,e_type_sfixed64,3},
+          {emqx_exhook_pb,e_type_sint,3},
+          {emqx_exhook_pb,m_overwrite,3},
+          {emqx_exhook_pb,v_ok,3},
+          {emqx_exproto_pb,e_type_bool,3},
+          {emqx_exproto_pb,e_type_double,3},
+          {emqx_exproto_pb,e_type_fixed32,3},
+          {emqx_exproto_pb,e_type_fixed64,3},
+          {emqx_exproto_pb,e_type_float,3},
+          {emqx_exproto_pb,e_type_int32,3},
+          {emqx_exproto_pb,e_type_int64,3},
+          {emqx_exproto_pb,e_type_sfixed32,3},
+          {emqx_exproto_pb,e_type_sfixed64,3},
+          {emqx_exproto_pb,e_type_sint,3},
+          {emqx_exproto_pb,m_overwrite,3},
+          {emqx_exproto_pb,v_ok,3},
+          {grpc_health_pb,cons,3},
+          {grpc_health_pb,e_type_bool,3},
+          {grpc_health_pb,e_type_bytes,3},
+          {grpc_health_pb,e_type_double,3},
+          {grpc_health_pb,e_type_fixed32,3},
+          {grpc_health_pb,e_type_fixed64,3},
+          {grpc_health_pb,e_type_float,3},
+          {grpc_health_pb,e_type_int32,3},
+          {grpc_health_pb,e_type_int64,3},
+          {grpc_health_pb,e_type_sfixed32,3},
+          {grpc_health_pb,e_type_sfixed64,3},
+          {grpc_health_pb,e_type_sint,3},
+          {grpc_health_pb,e_varint,3},
+          {grpc_health_pb,'erlang_++',3},
+          {grpc_health_pb,lists_reverse,2},
+          {grpc_health_pb,m_overwrite,3},
+          {grpc_health_pb,v_ok,3},
+          {grpc_reflection_pb,e_type_bool,3},
+          {grpc_reflection_pb,e_type_double,3},
+          {grpc_reflection_pb,e_type_fixed32,3},
+          {grpc_reflection_pb,e_type_fixed64,3},
+          {grpc_reflection_pb,e_type_float,3},
+          {grpc_reflection_pb,e_type_int64,3},
+          {grpc_reflection_pb,e_type_sfixed32,3},
+          {grpc_reflection_pb,e_type_sfixed64,3},
+          {grpc_reflection_pb,e_type_sint,3},
+          {grpc_reflection_pb,e_varint,3},
+          {grpc_reflection_pb,m_overwrite,3},
+          {grpc_reflection_pb,v_ok,3},
+          {prometheus_text_format,escape_metric_help,1},
+
+          %%
+          %% To be used in grammar files to throw an error message to the parser
+          %% toplevel. Doesn't have to be exported!
+          %%
+          {redbug_parser,return_error,2},
+          {rulesql,return_error,2},
+          {xmerl_b64Bin,return_error,2},
+          {xmerl_xpath_parse,return_error,2}]
+   }
+
+, #{ name => deprecated_function_calls
+   , analysis => deprecated_function_calls
+   , excl_apps =>
+         [
+         ]
+   , excl_mods =>
+         [
+         ]
+   , filters =>
+         [{{coap_core_link,join_uri,1},{http_uri,encode,1}},
+          {{disk_log_server,dist_pids,1},{pg2,get_members,1}},
+          {{disk_log_server,do_accessible_logs,0},{pg2,which_groups,0}},
+          {{disk_log_server,do_get_log_pids,1},{pg2,get_members,1}},
+          {{disk_log_server,do_open,3},{pg2,create,1}},
+          {{disk_log_server,erase_log,2},{pg2,leave,2}},
+          {{disk_log_server,handle_info,2},{pg2,join,2}},
+          {{gen_fsm,error_info,7},{gen_fsm,format_log,1}},
+          {{gen_fsm,error_info,7},{gen_fsm,format_log,2}},
+          {{gen_fsm,handle_msg,8},{gen_fsm,format_log,1}},
+          {{gen_fsm,handle_msg,8},{gen_fsm,format_log,2}},
+          {{httpd_util,decode_hex,1},{http_uri,decode,1}},
+          {{httpd_util,encode_hex,1},{http_uri,encode,1}},
+          {{igor,tidy,2},{erl_tidy,module,2}},
+          {{jose_public_key,pseudo_random_function,1},{crypto,hmac,4}},
+          {{lwm2m_coap_client,make_segment,1},{http_uri,decode,1}},
+          {{lwm2m_coap_client,resolve_uri,1},{http_uri,parse,2}},
+          {{lwm2m_coap_responder,cancel_observer,2},{pg2,delete,1}},
+          {{lwm2m_coap_responder,cancel_observer,2},{pg2,get_members,1}},
+          {{lwm2m_coap_responder,cancel_observer,2},{pg2,leave,2}},
+          {{lwm2m_coap_responder,handle_observe,4},{pg2,create,1}},
+          {{lwm2m_coap_responder,handle_observe,4},{pg2,join,2}},
+          {{lwm2m_coap_responder,notify,2},{pg2,get_members,1}},
+          {{ranch_ssl,handshake,3},{ssl,ssl_accept,3}}]
+   }
+
+, #{ name => deprecated_functions
+   , analysis => deprecated_functions
+   , excl_apps =>
+         [
+         ]
+   , excl_mods =>
+         [
+         ]
+   , filters =>
+         [{crypto,hmac,4},
+          {erl_tidy,module,2},
+          {gen_fsm,format_log,1},
+          {gen_fsm,format_log,2},
+          {http_uri,decode,1},
+          {http_uri,encode,1},
+          {http_uri,parse,2},
+          {pg2,create,1},
+          {pg2,delete,1},
+          {pg2,get_members,1},
+          {pg2,join,2},
+          {pg2,leave,2},
+          {pg2,which_groups,0},
+          {ssl,ssl_accept,3}]
+   }
+].

src/emqx.app.src

@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.21"}, % strict semver, bump manually!
+  {vsn, "4.3.22"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel

src/emqx.appup.src

@@ -1,30 +1,84 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.20",
+  [{"4.3.21",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]}]},
+   {"4.3.20",
+    [{add_module,emqx_secret},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]}]},
    {"4.3.19",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -35,9 +89,18 @@
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
@@ -55,9 +118,17 @@
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]}]},
    {"4.3.15",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {add_module,emqx_calendar},
@@ -83,9 +154,17 @@
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
      {update,emqx_os_mon,{advanced,[]}},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.14",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -113,9 +192,15 @@
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
      {update,emqx_os_mon,{advanced,[]}},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]},
    {"4.3.13",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
@@ -148,9 +233,15 @@
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
      {update,emqx_os_mon,{advanced,[]}},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -186,9 +277,15 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -228,7 +325,12 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -268,7 +370,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -312,7 +418,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -356,7 +466,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -400,7 +514,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -444,7 +562,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -488,7 +610,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -532,7 +658,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -576,7 +706,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -620,7 +754,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -666,7 +804,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
-    [{add_module,emqx_calendar},
+    [{add_module,emqx_secret},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -715,30 +857,80 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.20",
+  [
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+   {"4.3.21",
+    [{load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]}]},
+   {"4.3.20",
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]}]},
    {"4.3.19",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -749,9 +941,17 @@
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
@@ -769,9 +969,16 @@
      {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {delete_module,emqx_exclusive_subscription}]},
    {"4.3.15",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {delete_module,emqx_calendar},
@@ -796,9 +1003,16 @@
      {load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.14",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
@@ -825,9 +1039,14 @@
      {load_module,emqx_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]},
    {"4.3.13",
-    [{load_module,emqx_message,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
@@ -859,9 +1078,14 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -895,9 +1119,14 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -935,7 +1164,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -973,7 +1206,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1015,7 +1251,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1057,7 +1296,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1099,7 +1341,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1141,7 +1386,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1183,7 +1431,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1225,7 +1476,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1267,7 +1521,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1309,7 +1566,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -1353,7 +1613,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]},
+     {load_module,emqx_router,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tracer,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},

src/emqx.erl

@@ -228,6 +228,7 @@ shutdown() ->
 
 shutdown(Reason) ->
     ?LOG(critical, "emqx shutdown for ~s", [Reason]),
+    on_shutdown(Reason),
     _ = emqx_plugins:unload(),
     lists:foreach(fun application:stop/1
                  , lists:reverse(default_started_applications())
@@ -238,10 +239,12 @@ reboot() ->
         true ->
             _ = application:stop(emqx_dashboard), %% dashboard must be started after mnesia
             lists:foreach(fun application:start/1 , default_started_applications()),
-            application:start(emqx_dashboard);
+            _ = application:start(emqx_dashboard),
+            on_reboot();
 
         false ->
-            lists:foreach(fun application:start/1 , default_started_applications())
+            lists:foreach(fun application:start/1 , default_started_applications()),
+            on_reboot()
     end.
 
 is_application_running(App) ->
@@ -256,6 +259,32 @@ default_started_applications() ->
     [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules].
 -endif.
 
+-ifdef(EMQX_ENTERPRISE).
+on_reboot() ->
+    try
+        _ = emqx_license_api:bootstrap_license(),
+        ok
+    catch
+        Kind:Reason:Stack ->
+            ?LOG(critical, "~p while rebooting: ~p, ~p", [Kind, Reason, Stack]),
+            ok
+    end,
+    ok.
+
+on_shutdown(join) ->
+    emqx_modules:sync_load_modules_file(),
+    ok;
+on_shutdown(_) ->
+    ok.
+
+-else.
+on_reboot() ->
+    ok.
+
+on_shutdown(_) ->
+    ok.
+-endif.
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

src/emqx_alarm.erl

@@ -191,10 +191,8 @@ init([Opts]) ->
                                     size_limit = SizeLimit,
                                     validity_period = ValidityPeriod})}.
 
-%% suppress dialyzer warning due to dirty read/write race condition.
 %% TODO: change from dirty_read/write to transactional.
 %% TODO: handle mnesia write errors.
--dialyzer([{nowarn_function, [handle_call/3]}]).
 handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Actions}) ->
     case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of
         [#activated_alarm{name = Name}] ->

src/emqx_channel.erl

@@ -87,7 +87,7 @@
           %% Quota checkers
           quota :: maybe(emqx_limiter:limiter()),
           %% Timers
-          timers :: #{atom() => disabled | maybe(reference())},
+          timers :: #{channel_timer() => disabled | maybe(reference())},
           %% Conn State
           conn_state :: conn_state(),
           %% Takeover
@@ -109,6 +109,13 @@
 
 -type(replies() :: emqx_types:packet() | reply() | [reply()]).
 
+-type(channel_timer() :: alive_timer
+                       | retry_timer
+                       | await_timer
+                       | expire_timer
+                       | will_timer
+                       | quota_timer).
+
 -define(IS_MQTT_V5, #channel{conninfo = #{proto_ver := ?MQTT_PROTO_V5}}).
 
 -define(TIMER_TABLE, #{
@@ -122,8 +129,6 @@
 
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
 
--dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}).
-
 %%--------------------------------------------------------------------
 %% Info, Attrs and Caps
 %%--------------------------------------------------------------------
@@ -244,7 +249,6 @@ setting_peercert_infos(Peercert, ClientInfo, Options) ->
     ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN),
     ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
 
--dialyzer([{nowarn_function, [peer_cert_as/5]}]).
 % esockd_peercert:peercert is opaque
 % https://github.com/emqx/esockd/blob/master/src/esockd_peercert.erl
 peer_cert_as(Key, Options, Peercert, DN, CN) ->
@@ -947,9 +951,10 @@ return_sub_unsub_ack(Packet, Channel) ->
 handle_call(kick, Channel = #channel{
                                conn_state = ConnState,
                                will_msg = WillMsg,
+                               clientinfo = ClientInfo,
                                conninfo = #{proto_ver := ProtoVer}
                               }) ->
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+    (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
     Channel1 = case ConnState of
                    connected -> ensure_disconnected(kicked, Channel);
                    _ -> Channel
@@ -1098,8 +1103,9 @@ handle_timeout(_TRef, expire_awaiting_rel,
 handle_timeout(_TRef, expire_session, Channel) ->
     shutdown(expired, Channel);
 
-handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) ->
+handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg,
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+                                                       clientinfo = ClientInfo}) ->
+    (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
     {ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})};
 
 handle_timeout(_TRef, expire_quota_limit, Channel) ->
@@ -1113,12 +1119,8 @@ handle_timeout(_TRef, Msg, Channel) ->
 %% Ensure timers
 %%--------------------------------------------------------------------
 
-ensure_timer([Name], Channel) ->
+-spec ensure_timer(channel_timer(), channel()) -> channel().
-    ensure_timer(Name, Channel);
+ensure_timer(Name, Channel = #channel{timers = Timers}) when is_atom(Name) ->
-ensure_timer([Name | Rest], Channel) ->
-    ensure_timer(Rest, ensure_timer(Name, Channel));
-
-ensure_timer(Name, Channel = #channel{timers = Timers}) ->
     TRef = maps:get(Name, Timers, undefined),
     Time = interval(Name, Channel),
     case TRef == undefined andalso Time > 0 of
@@ -1126,6 +1128,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) ->
         false -> Channel %% Timer disabled or exists
     end.
 
+-spec ensure_timer(channel_timer(), timeout(), channel()) -> channel().
 ensure_timer(Name, Time, Channel = #channel{timers = Timers}) ->
     Msg = maps:get(Name, ?TIMER_TABLE),
     TRef = emqx_misc:start_timer(Time, Msg),
@@ -1140,36 +1143,48 @@ reset_timer(Name, Time, Channel) ->
 clean_timer(Name, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = maps:remove(Name, Timers)}.
 
+-spec interval(channel_timer(), channel()) -> timeout().
 interval(alive_timer, #channel{keepalive = KeepAlive}) ->
     emqx_keepalive:info(interval, KeepAlive);
 interval(retry_timer, #channel{session = Session}) ->
     timer:seconds(emqx_session:info(retry_interval, Session));
 interval(await_timer, #channel{session = Session}) ->
-    timer:seconds(emqx_session:info(await_rel_timeout, Session));
+    timer:seconds(emqx_session:info(await_rel_timeout, Session)).
-interval(expire_timer, #channel{conninfo = ConnInfo}) ->
-    timer:seconds(maps:get(expiry_interval, ConnInfo));
-interval(will_timer, #channel{will_msg = WillMsg}) ->
-    timer:seconds(will_delay_interval(WillMsg)).
 
 %%--------------------------------------------------------------------
 %% Terminate
 %%--------------------------------------------------------------------
 
 -spec(terminate(any(), channel()) -> ok).
-terminate(_, #channel{conn_state = idle}) -> ok;
+terminate(_Reason, #channel{conn_state = idle} = _Channel) ->
+    ?tp(channel_terminated, #{channel => _Channel, reason => _Reason}),
+    ok;
 terminate(normal, Channel) ->
     run_terminate_hook(normal, Channel);
-terminate({shutdown, Reason}, Channel)
+terminate(Reason, Channel = #channel{will_msg = WillMsg,
-  when Reason =:= kicked; Reason =:= discarded; Reason =:= takeovered ->
+                                     clientinfo = ClientInfo}) ->
-    run_terminate_hook(Reason, Channel);
+    should_publish_will_message(Reason, Channel)
-terminate(Reason, Channel = #channel{will_msg = WillMsg}) ->
+        andalso publish_will_msg(ClientInfo, WillMsg),
-    (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
     run_terminate_hook(Reason, Channel).
 
-run_terminate_hook(_Reason, #channel{session = undefined}) -> ok;
+run_terminate_hook(_Reason, #channel{session = undefined} = _Channel) ->
-run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session}) ->
+    ?tp(channel_terminated, #{channel => _Channel, reason => _Reason}),
+    ok;
+run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session} = _Channel) ->
+    ?tp(channel_terminated, #{channel => _Channel, reason => Reason}),
     emqx_session:terminate(ClientInfo, Reason, Session).
 
+should_publish_will_message(TerminateReason, Channel) ->
+    not lists:member(TerminateReason, [ {shutdown, kicked}
+                                      , {shutdown, discarded}
+                                      , {shutdown, takeovered}
+                                      , {shutdown, not_authorized}
+                                      ])
+        andalso not lists:member(info(conn_state, Channel), [ idle
+                                                            , connecting
+                                                            ])
+        andalso info(will_msg, Channel) =/= undefined.
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
@@ -1689,10 +1704,11 @@ ensure_disconnected(Reason, Channel = #channel{conninfo = ConnInfo,
 
 maybe_publish_will_msg(Channel = #channel{will_msg = undefined}) ->
     Channel;
-maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg}) ->
+maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg,
+                                          clientinfo = ClientInfo}) ->
     case will_delay_interval(WillMsg) of
         0 ->
-            ok = publish_will_msg(WillMsg),
+            ok = publish_will_msg(ClientInfo, WillMsg),
             Channel#channel{will_msg = undefined};
         I ->
             ensure_timer(will_timer, timer:seconds(I), Channel)
@@ -1702,9 +1718,19 @@ will_delay_interval(WillMsg) ->
     maps:get('Will-Delay-Interval',
              emqx_message:get_header(properties, WillMsg, #{}), 0).
 
-publish_will_msg(Msg) ->
+publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
+    case emqx_access_control:check_acl(ClientInfo, publish, Topic) of
+        allow ->
             _ = emqx_broker:publish(Msg),
-    ok.
+            ok;
+        deny ->
+            ?tp(
+                warning,
+                last_will_testament_publish_denied,
+                #{topic => Topic}
+            ),
+            ok
+    end.
 
 %%--------------------------------------------------------------------
 %% Disconnect Reason
@@ -1757,8 +1783,6 @@ shutdown(success, Reply, Channel) ->
 shutdown(Reason, Reply, Channel) ->
     {shutdown, Reason, Reply, Channel}.
 
-shutdown(success, Reply, Packet, Channel) ->
-    shutdown(normal, Reply, Packet, Channel);
 shutdown(Reason, Reply, Packet, Channel) ->
     {shutdown, Reason, Reply, Packet, Channel}.
 

src/emqx_connection.erl

@@ -125,12 +125,6 @@
 -define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]).
 
 -dialyzer({no_match, [info/2]}).
--dialyzer({nowarn_function, [ init/4
-                            , init_state/3
-                            , run_loop/2
-                            , system_terminate/4
-                            , system_code_change/4
-                            ]}).
 
 -spec(start_link(esockd:transport(), esockd:socket(), proplists:proplist())
       -> {ok, pid()}).
@@ -286,7 +280,7 @@ run_loop(Parent, State = #state{transport = Transport,
                                 peername  = Peername,
                                 channel   = Channel}) ->
     emqx_logger:set_metadata_peername(esockd:format(Peername)),
-    emqx_misc:tune_heap_size(emqx_zone:oom_policy(
+    _ = emqx_misc:tune_heap_size(emqx_zone:oom_policy(
                                    emqx_channel:info(zone, Channel))),
     case activate_socket(State) of
         {ok, NState} -> hibernate(Parent, NState);

src/emqx_frame.erl

@@ -78,8 +78,6 @@
 %% 16#0D,16#0A, 16#0D,16#0A,16#00,16#0D,16#0A,16#51,16#55,16#49,16#54,16#0A
 -define(PPV2_HEADER_SIG, "\r\n\r\n\0\r\nQUIT\n").
 
--dialyzer({no_match, [serialize_utf8_string/2]}).
-
 -ifdef(TEST).
 -export([parse_variable_byte_integer/1]).
 -endif.
@@ -788,8 +786,6 @@ serialize_utf8_pair({Name, Value}) ->
 serialize_binary_data(Bin) ->
     [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin].
 
-serialize_utf8_string(undefined, false) ->
-    error(utf8_string_undefined);
 serialize_utf8_string(undefined, true) ->
     <<>>;
 serialize_utf8_string(String, _AllowNull) ->

src/emqx_http_lib.erl

@@ -96,8 +96,8 @@ do_parse(URI) ->
 %% underscores replaced with hyphens
 %% NOTE: assuming the input Headers list is a proplists,
 %% that is, when a key is duplicated, list header overrides tail
-%% e.g. [{"Content_Type", "applicaiton/binary"}, {<<"content-type">>, "applicaiton/json"}]
+%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}]
-%% results in: [{"content-type", "applicaiton/binary"}]
+%% results in: [{<<"content-type">>, "applicaiton/binary"}]
 normalise_headers(Headers0) ->
     F = fun({K0, V}) ->
                 K = re:replace(K0, "_", "-", [{return,binary}]),

src/emqx_limiter.erl

@@ -55,8 +55,6 @@
 
 -type(limiter() :: #limiter{}).
 
--dialyzer({nowarn_function, [consume/3]}).
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------

src/emqx_misc.erl

@@ -45,6 +45,9 @@
         , index_of/2
         , maybe_parse_ip/1
         , ipv6_probe/1
+        , ipv6_probe/2
+        , pmap/2
+        , pmap/3
         ]).
 
 -export([ bin2hexstr_A_F/1
@@ -55,7 +58,13 @@
 -export([ is_sane_id/1
         ]).
 
+-export([
+    nolink_apply/1,
+    nolink_apply/2
+]).
+
 -define(VALID_STR_RE, "^[A-Za-z0-9]+[A-Za-z0-9-_]*$").
+-define(DEFAULT_PMAP_TIMEOUT, 5000).
 
 -spec is_sane_id(list() | binary()) -> ok | {error, Reason::binary()}.
 is_sane_id(Str) ->
@@ -84,12 +93,15 @@ maybe_parse_ip(Host) ->
 
 %% @doc Add `ipv6_probe' socket option if it's supported.
 ipv6_probe(Opts) ->
+    ipv6_probe(Opts, true).
+
+ipv6_probe(Opts, Ipv6Probe) when is_boolean(Ipv6Probe) orelse is_integer(Ipv6Probe) ->
     Bool = try gen_tcp:ipv6_probe()
            catch _ : _ -> false end,
-    ipv6_probe(Bool, Opts).
+    ipv6_probe(Bool, Opts, Ipv6Probe).
 
-ipv6_probe(false, Opts) -> Opts;
+ipv6_probe(false, Opts, _) -> Opts;
-ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts].
+ipv6_probe(true, Opts, Ipv6Probe) -> [{ipv6_probe, Ipv6Probe} | Opts].
 
 %% @doc Merge options
 -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()).
@@ -328,6 +340,110 @@ hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0;
 hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10;
 hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10.
 
+%% @doc Like lists:map/2, only the callback function is evaluated
+%% concurrently.
+-spec pmap(fun((A) -> B), list(A)) -> list(B).
+pmap(Fun, List) when is_function(Fun, 1), is_list(List) ->
+    pmap(Fun, List, ?DEFAULT_PMAP_TIMEOUT).
+
+-spec pmap(fun((A) -> B), list(A), timeout()) -> list(B).
+pmap(Fun, List, Timeout) when
+    is_function(Fun, 1), is_list(List), is_integer(Timeout), Timeout >= 0
+->
+    nolink_apply(fun() -> do_parallel_map(Fun, List) end, Timeout).
+
+%% @doc Delegate a function to a worker process.
+%% The function may spawn_link other processes but we do not
+%% want the caller process to be linked.
+%% This is done by isolating the possible link with a not-linked
+%% middleman process.
+nolink_apply(Fun) -> nolink_apply(Fun, infinity).
+
+%% @doc Same as `nolink_apply/1', with a timeout.
+-spec nolink_apply(function(), timer:timeout()) -> term().
+nolink_apply(Fun, Timeout) when is_function(Fun, 0) ->
+    Caller = self(),
+    ResRef = make_ref(),
+    Middleman = erlang:spawn(make_middleman_fn(Caller, Fun, ResRef)),
+    receive
+        {ResRef, {normal, Result}} ->
+            Result;
+        {ResRef, {exception, {C, E, S}}} ->
+            erlang:raise(C, E, S);
+        {ResRef, {'EXIT', Reason}} ->
+            exit(Reason)
+    after Timeout ->
+        exit(Middleman, kill),
+        exit(timeout)
+    end.
+
+-spec make_middleman_fn(pid(), fun(() -> any()), reference()) -> fun(() -> no_return()).
+make_middleman_fn(Caller, Fun, ResRef) ->
+    fun() ->
+        process_flag(trap_exit, true),
+        CallerMRef = erlang:monitor(process, Caller),
+        Worker = erlang:spawn_link(make_worker_fn(Caller, Fun, ResRef)),
+        receive
+            {'DOWN', CallerMRef, process, _, _} ->
+                %% For whatever reason, if the caller is dead,
+                %% there is no reason to continue
+                exit(Worker, kill),
+                exit(normal);
+            {'EXIT', Worker, normal} ->
+                exit(normal);
+            {'EXIT', Worker, Reason} ->
+                %% worker exited with some reason other than 'normal'
+                _ = erlang:send(Caller, {ResRef, {'EXIT', Reason}}),
+                exit(normal)
+        end
+    end.
+
+-spec make_worker_fn(pid(), fun(() -> any()), reference()) -> fun(() -> no_return()).
+make_worker_fn(Caller, Fun, ResRef) ->
+    fun() ->
+        Res =
+            try
+                {normal, Fun()}
+            catch
+                C:E:S ->
+                    {exception, {C, E, S}}
+            end,
+        _ = erlang:send(Caller, {ResRef, Res}),
+        exit(normal)
+    end.
+
+do_parallel_map(Fun, List) ->
+    Parent = self(),
+    PidList = lists:map(
+        fun(Item) ->
+            erlang:spawn_link(
+                fun() ->
+                    Res =
+                        try
+                            {normal, Fun(Item)}
+                        catch
+                            C:E:St ->
+                                {exception, {C, E, St}}
+                        end,
+                    Parent ! {self(), Res}
+                end
+            )
+        end,
+        List
+    ),
+    lists:foldr(
+        fun(Pid, Acc) ->
+            receive
+                {Pid, {normal, Result}} ->
+                    [Result | Acc];
+                {Pid, {exception, {C, E, St}}} ->
+                    erlang:raise(C, E, St)
+            end
+        end,
+        [],
+        PidList
+    ).
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 

src/emqx_plugins.erl

@@ -24,6 +24,7 @@
 -export([init/0]).
 
 -export([ load/0
+        , force_load/0
         , load/1
         , unload/0
         , unload/1
@@ -41,10 +42,6 @@
 -compile(nowarn_export_all).
 -endif.
 
--dialyzer({no_match, [ plugin_loaded/2
-                     , plugin_unloaded/2
-                     ]}).
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -63,12 +60,17 @@ init() ->
 %% @doc Load all plugins when the broker started.
 -spec(load() -> ok | ignore | {error, term()}).
 load() ->
+    do_load(#{force_load => false}).
+force_load() ->
+    do_load(#{force_load => true}).
+
+do_load(Options) ->
     ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)),
     case emqx:get_env(plugins_loaded_file) of
         undefined -> ignore; %% No plugins available
         File ->
             _ = ensure_file(File),
-            with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end)
+            with_loaded_file(File, fun(Names) -> load_plugins(Names, Options, false) end)
     end.
 
 %% @doc Load a Plugin
@@ -105,7 +107,7 @@ unload(PluginName) when is_atom(PluginName) ->
             ?LOG(error, "Plugin ~s is not started", [PluginName]),
             {error, not_started};
         {_, _} ->
-            unload_plugin(PluginName, true)
+            unload_plugin(PluginName)
     end.
 
 reload(PluginName) when is_atom(PluginName)->
@@ -286,18 +288,23 @@ filter_plugins([{Name, Load} | Names], Plugins) ->
 filter_plugins([Name | Names], Plugins) when is_atom(Name) ->
     filter_plugins([{Name, true} | Names], Plugins).
 
-load_plugins(Names, Persistent) ->
+load_plugins(Names, Options, Persistent) ->
     Plugins = list(),
     NotFound = Names -- names(Plugins),
     case NotFound of
         []       -> ok;
         NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound])
     end,
-    NeedToLoad = (Names -- NotFound) -- names(started_app),
+    NeedToLoad0 = Names -- NotFound,
+    NeedToLoad1 =
+        case Options of
+            #{force_load := true} -> NeedToLoad0;
+            _ -> NeedToLoad0 -- names(started_app)
+        end,
     lists:foreach(fun(Name) ->
                       Plugin = find_plugin(Name, Plugins),
                       load_plugin(Plugin#plugin.name, Persistent)
-                  end, NeedToLoad).
+                  end, NeedToLoad1).
 
 generate_configs(App) ->
     ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config",
@@ -384,10 +391,11 @@ start_app(App, SuccFun) ->
             {error, {ErrApp, Reason}}
     end.
 
-unload_plugin(App, Persistent) ->
+unload_plugin(App) ->
     case stop_app(App) of
         ok ->
-            _ = plugin_unloaded(App, Persistent), ok;
+            _ = plugin_unloaded(App),
+            ok;
         {error, Reason} ->
             {error, Reason}
     end.
@@ -428,9 +436,7 @@ plugin_loaded(Name, true) ->
             ?LOG(error, "Cannot read loaded plugins: ~p", [Error])
     end.
 
-plugin_unloaded(_Name, false) ->
+plugin_unloaded(Name) ->
-    ok;
-plugin_unloaded(Name, true) ->
     case read_loaded() of
         {ok, Names0} ->
             Names = filter_plugins(Names0),

src/emqx_router.erl

@@ -266,9 +266,6 @@ maybe_trans(Fun, Args) ->
                   end, [])
     end.
 
-%% The created fun only terminates with explicit exception
--dialyzer({nowarn_function, [trans/2]}).
-
 -spec(trans(function(), list(any())) -> ok | {error, term()}).
 trans(Fun, Args) ->
     {WPid, RefMon} =
@@ -277,13 +274,7 @@ trans(Fun, Args) ->
             %% are caught by mnesia:transaction/2.
             %% Future changes should keep in mind that this process
             %% always exit with database write result.
-            fun() ->
+            make_trans(Fun, Args)),
-                    Res = case mnesia:transaction(Fun, Args) of
-                              {atomic, Ok} -> Ok;
-                              {aborted, Reason} -> {error, Reason}
-                          end,
-                    exit({shutdown, Res})
-            end),
     %% Receive a 'shutdown' exit to pass result from the short-lived process.
     %% so the receive below can be receive-mark optimized by the compiler.
     %%
@@ -300,6 +291,16 @@ trans(Fun, Args) ->
             end
     end.
 
+-spec make_trans(fun((...) -> term()), [term()]) -> fun(() -> no_return()).
+make_trans(Fun, Args) ->
+    fun() ->
+            Res = case mnesia:transaction(Fun, Args) of
+                      {atomic, Ok} -> Ok;
+                      {aborted, Reason} -> {error, Reason}
+                  end,
+            exit({shutdown, Res})
+    end.
+
 lock_router() ->
     %% if Retry is not 0, global:set_lock could sleep a random time up to 8s.
     %% Considering we have a limited number of brokers, it is safe to use sleep 1 ms.

src/emqx_router_helper.erl

@@ -53,8 +53,6 @@
 -define(ROUTING_NODE, emqx_routing_node).
 -define(LOCK, {?MODULE, cleanup_routes}).
 
--dialyzer({nowarn_function, [cleanup_routes/1]}).
-
 %%--------------------------------------------------------------------
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
@@ -176,4 +174,3 @@ cleanup_routes(Node) ->
                 #route{_ = '_', dest = {'_', Node}}],
     [mnesia:delete_object(?ROUTE, Route, write)
      || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)].
-

src/emqx_secret.erl

@@ -0,0 +1,41 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% Note: this module CAN'T be hot-patched to avoid invalidating the
+%% closures, so it must not be changed.
+-module(emqx_secret).
+
+%% API:
+-export([wrap/1, unwrap/1]).
+
+%%================================================================================
+%% API funcions
+%%================================================================================
+
+wrap(undefined) ->
+    undefined;
+wrap(Func) when is_function(Func) ->
+    Func;
+wrap(Term) ->
+    fun() ->
+        Term
+    end.
+
+unwrap(Term) when is_function(Term, 0) ->
+    %% Handle potentially nested funs
+    unwrap(Term());
+unwrap(Term) ->
+    Term.

src/emqx_tracer.erl

@@ -59,8 +59,6 @@
         L =:= info orelse
         L =:= debug).
 
--dialyzer({nowarn_function, [install_trace_handler/3]}).
-
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------

src/emqx_types.erl

@@ -201,7 +201,6 @@
 -type(deliver_result() :: ok | {ok, non_neg_integer()} | {error, term()}).
 -type(publish_result() :: [{node(), topic(), deliver_result()} |
                            {share, topic(), deliver_result()}]).
--type(route() :: #route{}).
 -type(sub_group() :: tuple() | binary()).
 -type(route_entry() :: {topic(), node()} | {topic, sub_group()}).
 -type(plugin() :: #plugin{}).
@@ -215,4 +214,3 @@
 -type(oom_policy() :: #{message_queue_len => non_neg_integer(),
                         max_heap_size => non_neg_integer()
                        }).
-

src/emqx_ws_connection.erl

@@ -100,7 +100,6 @@
 -define(ENABLED(X), (X =/= undefined)).
 
 -dialyzer({no_match, [info/2]}).
--dialyzer({nowarn_function, [websocket_init/1]}).
 
 %%--------------------------------------------------------------------
 %% Info, Stats
@@ -304,7 +303,7 @@ websocket_init([Req, Opts]) ->
     %% MQTT Idle Timeout
     IdleTimeout = emqx_zone:idle_timeout(Zone),
     IdleTimer = start_timer(IdleTimeout, idle_timeout),
-    emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)),
+    _ = emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)),
     emqx_logger:set_metadata_peername(esockd:format(Peername)),
     {ok, #state{peername       = Peername,
                 sockname       = Sockname,
@@ -777,4 +776,3 @@ get_peer(Req, Opts) ->
 set_field(Name, Value, State) ->
     Pos = emqx_misc:index_of(Name, record_info(fields, state)),
     setelement(Pos+1, State, Value).
-

test/emqx_misc_SUITE.erl

@@ -146,3 +146,36 @@ t_now_to_secs(_) ->
 t_now_to_ms(_) ->
     ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))).
 
+t_pmap_normal(_) ->
+    ?assertEqual(
+        [5, 7, 9],
+        emqx_misc:pmap(
+            fun({A, B}) -> A + B end,
+            [{2, 3}, {3, 4}, {4, 5}]
+        )
+    ).
+
+t_pmap_timeout(_) ->
+    ?assertExit(
+        timeout,
+        emqx_misc:pmap(
+            fun
+                (timeout) -> ct:sleep(1000);
+                ({A, B}) -> A + B
+            end,
+            [{2, 3}, {3, 4}, timeout],
+            100
+        )
+    ).
+
+t_pmap_exception(_) ->
+    ?assertError(
+        foobar,
+        emqx_misc:pmap(
+            fun
+                (error) -> error(foobar);
+                ({A, B}) -> A + B
+            end,
+            [{2, 3}, {3, 4}, error]
+        )
+    ).

test/emqx_node_helpers.erl

@@ -18,33 +18,57 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
--define(SLAVE_START_APPS, [emqx]).
+%% modules is included because code is called before cluster join
+-define(SLAVE_START_APPS, [emqx, emqx_modules]).
 
 -export([start_slave/1,
          start_slave/2,
-         stop_slave/1]).
+         stop_slave/1,
+         wait_for_synced_routes/3
+        ]).
 
 start_slave(Name) ->
     start_slave(Name, #{}).
 
 start_slave(Name, Opts) ->
-    {ok, Node} = ct_slave:start(list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
+    Node = make_node_name(Name),
-                                [{kill_if_fail, true},
+    case ct_slave:start(Node, [{kill_if_fail, true},
                                {monitor_master, true},
                                {init_timeout, 10000},
                                {startup_timeout, 10000},
-                                 {erl_flags, ebin_path()}]),
+                               {erl_flags, ebin_path()}]) of
-
+        {ok, _} ->
+            ok;
+        {error, started_not_connected, _} ->
+            ok
+    end,
     pong = net_adm:ping(Node),
     setup_node(Node, Opts),
     Node.
 
-stop_slave(Node) ->
+make_node_name(Name) ->
-    rpc:call(Node, ekka, leave, []),
+    case string:tokens(atom_to_list(Name), "@") of
-    ct_slave:stop(Node).
+        [_Name, _Host] ->
+            %% the name already has a @
+            Name;
+        _ ->
+            list_to_atom(atom_to_list(Name) ++ "@" ++ host())
+    end.
+
+stop_slave(Node0) ->
+    Node = make_node_name(Node0),
+    case rpc:call(Node, ekka, leave, []) of
+        ok -> ok;
+        {badrpc, nodedown} -> ok
+    end,
+    case ct_slave:stop(Node) of
+        {ok, _} -> ok;
+        {error, not_started, _} -> ok
+    end.
 
 host() ->
-    [_, Host] = string:tokens(atom_to_list(node()), "@"), Host.
+    [_, Host] = string:tokens(atom_to_list(node()), "@"),
+    Host.
 
 ebin_path() ->
     string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
@@ -71,7 +95,12 @@ setup_node(Node, #{} = Opts) ->
     [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx]],
     ok = rpc:call(Node, emqx_ct_helpers, start_apps, [StartApps, EnvHandler]),
 
-    rpc:call(Node, ekka, join, [node()]),
+    case maps:get(no_join, Opts, false) of
+        true ->
+            ok;
+        false ->
+            ok = rpc:call(Node, ekka, join, [node()])
+    end,
 
     %% Sanity check. Assert that `gen_rpc' is set up correctly:
     ?assertEqual( Node
@@ -81,3 +110,40 @@ setup_node(Node, #{} = Opts) ->
                 , gen_rpc:call(Node, gen_rpc, call, [node(), erlang, node, []])
                 ),
     ok.
+
+%% Routes are replicated async.
+%% Call this function to wait for nodes in the cluster to have the same view
+%% for a given topic.
+wait_for_synced_routes(Nodes, Topic, Timeout) ->
+    F = fun() -> do_wait_for_synced_routes(Nodes, Topic) end,
+    emqx_misc:nolink_apply(F, Timeout).
+
+do_wait_for_synced_routes(Nodes, Topic) ->
+    PerNodeView0 =
+        lists:map(
+          fun(Node) ->
+                  {rpc:call(Node, emqx_router, match_routes, [Topic]), Node}
+          end, Nodes),
+    PerNodeView = lists:keysort(1, PerNodeView0),
+    case check_consistent_view(PerNodeView) of
+        {ok, OneView} ->
+            ct:pal("consistent_routes_view~n~p", [OneView]),
+            ok;
+        {error, Reason}->
+            ct:pal("inconsistent_routes_view~n~p", [Reason]),
+            timer:sleep(10),
+            do_wait_for_synced_routes(Nodes, Topic)
+    end.
+
+check_consistent_view(PerNodeView) ->
+    check_consistent_view(PerNodeView, []).
+
+check_consistent_view([], [OneView]) -> {ok, OneView};
+check_consistent_view([], MoreThanOneView) -> {error, MoreThanOneView};
+check_consistent_view([{View, Node} | Rest], [{View, Nodes} | Acc]) ->
+    check_consistent_view(Rest, [{View, add_to_list(Node, Nodes)} | Acc]);
+check_consistent_view([{View, Node} | Rest], Acc) ->
+    check_consistent_view(Rest, [{View, Node} | Acc]).
+
+add_to_list(Node, Nodes) when is_list(Nodes) -> [Node | Nodes];
+add_to_list(Node, Node1) -> [Node, Node1].

test/emqx_plugins_SUITE.erl

@@ -160,8 +160,8 @@ t_plugin_loaded(_) ->
     ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)).
 
 t_plugin_unloaded(_) ->
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)),
+    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin)),
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)).
+    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin)).
 
 t_plugin(_) ->
     try
@@ -199,8 +199,8 @@ t_unload_plugin(_) ->
                                            (error_app) -> {error, error};
                                            (_) -> ok end),
 
-    ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app, true)),
+    ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app)),
-    ?assertEqual(ok, emqx_plugins:unload_plugin(normal, true)),
+    ?assertEqual(ok, emqx_plugins:unload_plugin(normal)),
-    ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app, true)),
+    ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app)),
 
     ok = meck:unload(application).

test/emqx_shared_sub_SUITE.erl

@@ -51,6 +51,9 @@ init_per_suite(Config) ->
     PortDiscovery = application:get_env(gen_rpc, port_discovery),
     application:set_env(gen_rpc, port_discovery, stateless),
     application:ensure_all_started(gen_rpc),
+    %% ensure emqx_moduels' app modules are loaded
+    %% so the mnesia tables are created
+    ok = load_app(emqx_modules),
     emqx_ct_helpers:start_apps([]),
     [{port_discovery, PortDiscovery} | Config].
 
@@ -61,32 +64,45 @@ end_per_suite(Config) ->
       _ -> ok
     end.
 
-t_is_ack_required(_) ->
+init_per_testcase(Case, Config) ->
+    try
+        ?MODULE:Case({'init', Config})
+    catch
+        error : function_clause ->
+            Config
+    end.
+
+end_per_testcase(Case, Config) ->
+    try
+        ?MODULE:Case({'end', Config})
+    catch
+        error : function_clause ->
+            ok
+    end.
+
+t_is_ack_required(Config) when is_list(Config) ->
     ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
 
-t_maybe_nack_dropped(_) ->
+t_maybe_nack_dropped(Config) when is_list(Config) ->
     ?assertEqual(store, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
     Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}},
     ?assertEqual(drop, emqx_shared_sub:maybe_nack_dropped(Msg)),
     ?assertEqual(ok,receive {for_test, {shared_sub_nack, dropped}} -> ok after 100 -> timeout end).
 
-t_nack_no_connection(_) ->
+t_nack_no_connection(Config) when is_list(Config) ->
     Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}},
     ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)),
     ?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok
                     after 100 -> timeout end).
 
-t_maybe_ack(_) ->
+t_maybe_ack(Config) when is_list(Config) ->
     ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})),
     Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}},
     ?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}},
                  emqx_shared_sub:maybe_ack(Msg)),
     ?assertEqual(ok,receive {for_test, ?ack} -> ok after 100 -> timeout end).
 
-% t_subscribers(_) ->
+t_random_basic(Config) when is_list(Config) ->
-%     error('TODO').
-
-t_random_basic(_) ->
     ok = ensure_config(random),
     ClientId = <<"ClientId">>,
     Topic = <<"foo">>,
@@ -116,7 +132,7 @@ t_random_basic(_) ->
 %% After the connection for the 2nd session is also closed,
 %% i.e. when all clients are offline, the following message(s)
 %% should be delivered randomly.
-t_no_connection_nack(_) ->
+t_no_connection_nack(Config) when is_list(Config) ->
     ok = ensure_config(sticky),
     Publisher = <<"publisher">>,
     Subscriber1 = <<"Subscriber1">>,
@@ -149,27 +165,27 @@ t_no_connection_nack(_) ->
     ?assertMatch([#{packet_id := 1}], recv_msgs(1)),
     ok.
 
-t_random(_) ->
+t_random(Config) when is_list(Config) ->
     ok = ensure_config(random, true),
     test_two_messages(random).
 
-t_round_robin(_) ->
+t_round_robin(Config) when is_list(Config) ->
     ok = ensure_config(round_robin, true),
     test_two_messages(round_robin).
 
-t_sticky(_) ->
+t_sticky(Config) when is_list(Config) ->
     ok = ensure_config(sticky, true),
     test_two_messages(sticky).
 
-t_hash(_) ->
+t_hash(Config) when is_list(Config) ->
     ok = ensure_config(hash, false),
     test_two_messages(hash).
 
-t_hash_clinetid(_) ->
+t_hash_clinetid(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
-t_hash_topic(_) ->
+t_hash_topic(Config) when is_list(Config) ->
     ok = ensure_config(hash_topic, false),
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
@@ -208,7 +224,7 @@ t_hash_topic(_) ->
     ok.
 
 %% if the original subscriber dies, change to another one alive
-t_not_so_sticky(_) ->
+t_not_so_sticky(Config) when is_list(Config) ->
     ok = ensure_config(sticky),
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
@@ -281,7 +297,7 @@ last_message(ExpectedPayload, Pids, Timeout) ->
         <<"not yet?">>
     end.
 
-t_dispatch(_) ->
+t_dispatch(Config) when is_list(Config) ->
     ok = ensure_config(random),
     Topic = <<"foo">>,
     ?assertEqual({error, no_subscribers},
@@ -290,18 +306,13 @@ t_dispatch(_) ->
     ?assertEqual({ok, 1},
                  emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})).
 
-% t_unsubscribe(_) ->
+t_uncovered_func(Config) when is_list(Config) ->
-%     error('TODO').
-
-% t_subscribe(_) ->
-%     error('TODO').
-t_uncovered_func(_) ->
     ignored = gen_server:call(emqx_shared_sub, ignored),
     ok = gen_server:cast(emqx_shared_sub, ignored),
     ignored = emqx_shared_sub ! ignored,
     {mnesia_table_event, []} = emqx_shared_sub ! {mnesia_table_event, []}.
 
-t_per_group_config(_) ->
+t_per_group_config(Config) when is_list(Config) ->
     ok = ensure_group_config(#{
                                <<"local_group_fallback">> => local,
                                <<"local_group">> => local,
@@ -320,8 +331,8 @@ t_per_group_config(_) ->
     test_two_messages(round_robin, <<"round_robin_group">>),
     test_two_messages(round_robin, <<"round_robin_group">>).
 
-t_local(_) ->
+t_local({'init', Config}) ->
-    Node = start_slave('local_shared_sub_test19', 21884),
+    Node = start_slave(local_shared_sub_test19, 21884),
     GroupConfig = #{
                     <<"local_group_fallback">> => local,
                     <<"local_group">> => local,
@@ -330,7 +341,11 @@ t_local(_) ->
                    },
     ok = ensure_group_config(Node, GroupConfig),
     ok = ensure_group_config(GroupConfig),
-
+    [{slave_node, Node} | Config];
+t_local({'end', _Config}) ->
+    ok = stop_slave(local_shared_sub_test19);
+t_local(Config) when is_list(Config) ->
+    Node = proplists:get_value(slave_node, Config),
     Topic = <<"local_foo1/bar">>,
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
@@ -366,7 +381,7 @@ t_local(_) ->
     ?assertNotEqual(UsedSubPid1, UsedSubPid2),
     ok.
 
-t_local_fallback(_) ->
+t_local_fallback({'init', Config}) ->
     ok = ensure_group_config(#{
                                <<"local_group_fallback">> => local,
                                <<"local_group">> => local,
@@ -374,10 +389,15 @@ t_local_fallback(_) ->
                                <<"sticky_group">> => sticky
                               }),
 
+    Node = start_slave(local_fallback_shared_sub_test19, 11885),
+    [{slave_node, Node} | Config];
+t_local_fallback({'end', _}) ->
+    ok = stop_slave(local_fallback_shared_sub_test19);
+t_local_fallback(Config) when is_list(Config) ->
     Topic = <<"local_foo2/bar">>,
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
-    Node = start_slave('local_fallback_shared_sub_test19', 11885),
+    Node = proplists:get_value(slave_node, Config),
 
     {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]),
     {ok, _} = emqtt:connect(ConnPid1),
@@ -385,6 +405,7 @@ t_local_fallback(_) ->
     Message2 = emqx_message:make(ClientId2, 0, Topic, <<"hello2">>),
 
     emqtt:subscribe(ConnPid1, {<<"$share/local_group_fallback/", Topic/binary>>, 0}),
+    ok = emqx_node_helpers:wait_for_synced_routes([node(), Node], Topic, timer:seconds(10)),
 
     [{share, Topic, {ok, 1}}] = emqx:publish(Message1),
     {true, UsedSubPid1} = last_message(<<"hello1">>, [ConnPid1]),
@@ -400,10 +421,10 @@ t_local_fallback(_) ->
 
 %% This one tests that broker tries to select another shared subscriber
 %% If the first one doesn't return an ACK
-t_redispatch_with_ack(Config) ->
+t_redispatch_with_ack(Config) when is_list(Config) ->
     test_redispatch(Config, true).
 
-t_redispatch_no_ack(Config) ->
+t_redispatch_no_ack(Config) when is_list(Config) ->
     test_redispatch(Config, false).
 
 test_redispatch(_Config, AckEnabled) ->
@@ -437,10 +458,10 @@ test_redispatch(_Config, AckEnabled) ->
     emqtt:stop(UsedSubPid2),
     ok.
 
-t_redispatch_wildcard_with_ack(Config) ->
+t_redispatch_wildcard_with_ack(Config) when is_list(Config)->
     redispatch_wildcard(Config, true).
 
-t_redispatch_wildcard_no_ack(Config) ->
+t_redispatch_wildcard_no_ack(Config) when is_list(Config) ->
     redispatch_wildcard(Config, false).
 
 %% This one tests that broker tries to redispatch to another member in the group
@@ -475,15 +496,19 @@ redispatch_wildcard(_Config, AckEnabled) ->
     emqtt:stop(UsedSubPid2),
     ok.
 
-t_dispatch_when_inflights_are_full_with_ack(Config) when is_list(Config) ->
+t_dispatch_when_inflights_are_full({init, Config}) ->
+    %% make sure broker does not push more than one inflight
+    meck:new(emqx_zone, [passthrough, no_history]),
+    meck:expect(emqx_zone, max_inflight, fun(_Zone) -> 1 end),
+    Config;
+t_dispatch_when_inflights_are_full({'end', _Config}) ->
+    meck:unload(emqx_zone);
+t_dispatch_when_inflights_are_full(Config) when is_list(Config) ->
     ok = ensure_config(round_robin, _AckEnabled = true),
     Topic = <<"foo/bar">>,
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
 
-    %% make sure broker does not push more than one inflight
-    meck:new(emqx_zone, [passthrough, no_history]),
-    meck:expect(emqx_zone, max_inflight, fun(_Zone) -> 1 end),
     {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]),
     {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}]),
     {ok, _} = emqtt:connect(ConnPid1),
@@ -520,7 +545,6 @@ t_dispatch_when_inflights_are_full_with_ack(Config) when is_list(Config) ->
     ?assertMatch({true, ConnPid2}, last_message(<<"hello3">>, [ConnPid1, ConnPid2])),
     ?assertMatch({true, ConnPid2}, last_message(<<"hello4">>, [ConnPid1, ConnPid2])),
 
-    meck:unload(emqx_zone),
     emqtt:stop(ConnPid2),
     ok.
 
@@ -529,13 +553,17 @@ t_dispatch_when_inflights_are_full_with_ack(Config) when is_list(Config) ->
 %% client2 acts normal (aot_ack=true)
 %% Expected behaviour:
 %% the messages sent to client1's inflight and mq are re-dispatched after client1 is down
+t_dispatch_qos2({init, Config}) when is_list(Config) ->
+    meck:new(emqx_zone, [passthrough, no_history]),
+    meck:expect(emqx_zone, max_inflight, fun(_Zone) -> 1 end),
+    Config;
+t_dispatch_qos2({'end', Config}) when is_list(Config) ->
+    meck:unload(emqx_zone);
 t_dispatch_qos2(Config) when is_list(Config) ->
     ok = ensure_config(round_robin, _AckEnabled = false),
     Topic = <<"foo/bar/1">>,
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
-    meck:new(emqx_zone, [passthrough, no_history]),
-    meck:expect(emqx_zone, max_inflight, fun(_Zone) -> 1 end),
 
     {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
     {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
@@ -579,7 +607,6 @@ t_dispatch_qos2(Config) when is_list(Config) ->
     %% assert hello2 > hello1 or hello4 > hello3
     ?assert(MsgRec4 > MsgRec3),
     emqtt:stop(ConnPid2),
-    meck:unload(emqx_zone),
     ok.
 
 %%--------------------------------------------------------------------
@@ -636,8 +663,20 @@ recv_msgs(Count, Msgs) ->
     end.
 
 start_slave(Name, Port) ->
+    ok = emqx_ct_helpers:start_apps([emqx_modules]),
     Listeners = [#{listen_on => {{127,0,0,1}, Port},
+                   start_apps => [emqx, emqx_modules],
                    name => "internal",
                    opts => [{zone,internal}],
                    proto => tcp}],
     emqx_node_helpers:start_slave(Name, #{listeners => Listeners}).
+
+stop_slave(Name) ->
+    emqx_node_helpers:stop_slave(Name).
+
+load_app(App) ->
+    case application:load(App) of
+        ok -> ok;
+        {error, {already_loaded, _}} -> ok;
+        {error, Reason} -> error({failed_to_load_app, App, Reason})
+    end.