From 0c2ba9fe50e4cd7184135fa217df0d6063be44af Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Sat, 26 Feb 2022 21:30:32 +0100
Subject: [PATCH 01/17] fix(helm): fix configmap.env template

fix: #7070
---
 deploy/charts/emqx/templates/configmap.env.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index 851a7496b..ff5924c54 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
   {{- range $index, $value := .Values.emqxConfig }}
-  {{- if ne $value nil }}
+  {{- if $value }}
   {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
   {{ print "EMQX_" $key }}: {{ $value | quote }}
   {{- end }}

From 06db325b5fdf1c1ee5d2299e3a0e5c5654d95f23 Mon Sep 17 00:00:00 2001
From: ieQu1 <99872536+ieQu1@users.noreply.github.com>
Date: Fri, 25 Feb 2022 11:43:20 +0100
Subject: [PATCH 02/17] fix(auth_mnesia): Fix error message

---
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src   | 2 +-
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 4 ++++
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl       | 5 +++--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
index d782f0272..d8968c05f 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
@@ -1,6 +1,6 @@
 {application, emqx_auth_mnesia,
  [{description, "EMQ X Authentication with Mnesia"},
-  {vsn, "4.3.5"}, % strict semver, bump manually
+  {vsn, "4.3.6"}, % strict semver, bump manually
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,mnesia]},
diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
index acaf9403d..9ae492c5f 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
@@ -17,6 +17,8 @@
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.5">>,
+    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{<<"4.3.[0-3]">>,
     [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
@@ -35,4 +37,6 @@
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.5">>,
+    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.
diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl
index 72b9cfd2b..386adba4b 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl
@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -72,7 +72,8 @@ check(ClientInfo = #{ clientid := Clientid
         List ->
             case match_password(NPassword, HashType, List)  of
                 false ->
-                    ?LOG(error, "[Mnesia] Auth from mnesia failed: ~p", [ClientInfo]),
+                    Info = maps:without([password], ClientInfo),
+                    ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]),
                     emqx_metrics:inc(?AUTH_METRICS(failure)),
                     {stop, AuthResult#{anonymous => false, auth_result => password_error}};
                 _ ->

From 71100427ddaea1f86df7ca863ed81594c70a8853 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 1 Mar 2022 10:13:05 +0800
Subject: [PATCH 03/17] fix(relup): release upgrade failed if release dir is
 removed

If the target vsn is in `unpacked` state, but the release dir
`releases/<vsn>` is removed, then installing the target vsn
again will fail with error `{error,enoent}`.

In this case we should extract the tar ball again and then continue
to install the release.
---
 bin/install_upgrade.escript | 41 +++++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript
index 0bd4bdee1..ae89ea724 100755
--- a/bin/install_upgrade.escript
+++ b/bin/install_upgrade.escript
@@ -144,9 +144,10 @@ parse_arguments([VersionStr|Rest], Acc) ->
     parse_arguments(Rest, [{version, Version}] ++ Acc).
 
 unpack_release(RelName, TargetNode, Version) ->
+    StartScriptExists = filelib:is_dir(filename:join(["releases", Version, "start.boot"])),
     WhichReleases = which_releases(TargetNode),
     case proplists:get_value(Version, WhichReleases) of
-        undefined ->
+        Res when Res =:= undefined; (Res =:= unpacked andalso not StartScriptExists) ->
             %% not installed, so unpack tarball:
             %% look for a release package with the intended version in the following order:
             %%      releases/<relname>-<version>.tar.gz
@@ -161,10 +162,39 @@ unpack_release(RelName, TargetNode, Version) ->
                     case rpc:call(TargetNode, release_handler, unpack_release,
                                   [ReleasePackageLink], ?TIMEOUT) of
                         {ok, Vsn} -> {ok, Vsn};
+                        {error, {existing_release, Vsn}} ->
+                            %% sometimes the user may have removed the release/<vsn> dir
+                            %% for an `unpacked` release, then we need to re-unpack it from
+                            %% the .tar ball
+                            untar_for_unpacked_release(str(RelName), Vsn),
+                            {ok, Vsn};
                         {error, _} = Error -> Error
                     end
             end;
-        Other -> Other
+        Other ->
+            Other
+    end.
+
+untar_for_unpacked_release(RelName, Vsn) ->
+    {ok, Root} = file:get_cwd(),
+    RelDir = filename:join([Root, "releases"]),
+    %% untar the .tar file, so release/<vsn> will be created
+    Tar = filename:join([RelDir, Vsn, RelName ++ ".tar.gz"]),
+    extract_tar(Root, Tar),
+
+    %% create RELEASE file
+    RelFile = filename:join([RelDir, Vsn, RelName ++ ".rel"]),
+    release_handler:create_RELEASES(Root, RelFile),
+
+    %% Clean release
+    _ = file:delete(Tar),
+    _ = file:delete(RelFile).
+
+extract_tar(Cwd, Tar) ->
+    case erl_tar:extract(Tar, [keep_old_files, {cwd, Cwd}, compressed]) of
+        ok -> ok;
+        {error, {Name, Reason}} ->		% New erl_tar (R3A).
+            throw({error, {cannot_extract_file, Name, Reason}})
     end.
 
 %% 1. look for a release package tarball with the provided version in the following order:
@@ -391,3 +421,10 @@ erts_vsn() ->
     {ok, Str} = file:read_file(filename:join(["releases", "start_erl.data"])),
     [ErtsVsn, _] = string:tokens(binary_to_list(Str), " "),
     ErtsVsn.
+
+str(A) when is_atom(A) ->
+    atom_to_list(A);
+str(A) when is_binary(A) ->
+    binary_to_list(A);
+str(A) when is_list(A) ->
+    (A).

From ed41a66c6e89bd05955612290eadab22a6c01c25 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 25 Feb 2022 11:15:59 +0800
Subject: [PATCH 04/17] fix: rpc get node info maybe crash when emqx_sys
 process is not start yet.

---
 CHANGES-4.3.md     |  5 +++++
 src/emqx.appup.src | 50 ++++++++++++++++++++++++++++++++++++----------
 src/emqx_sys.erl   | 35 +++++++++++---------------------
 3 files changed, 57 insertions(+), 33 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 648086ec4..0e36babca 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -10,6 +10,11 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.14
+
+### Bug fixes
+* Fix rpc get node info maybe crash when other nodes is not ready.
+
 ## v4.3.13
 
 ### Important changes
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index dc045e08a..14ef02f80 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,8 +1,10 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.13",
-    [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+  [
+    {"4.3.13",
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
@@ -10,8 +12,9 @@
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -30,6 +33,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -52,6 +56,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -74,6 +79,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -100,6 +106,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -126,6 +133,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -154,6 +162,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
     [{load_module,emqx_pmon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -182,6 +191,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -211,6 +221,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -241,6 +252,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -272,6 +284,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -303,6 +316,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -338,6 +352,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
@@ -375,9 +390,11 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.13",
-    [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+  [
+   {"4.3.13",
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -385,8 +402,9 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
-    [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -404,6 +422,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -425,6 +444,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -446,6 +466,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -471,6 +492,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -496,6 +518,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -523,6 +546,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -550,6 +574,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -578,6 +603,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -607,6 +633,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -637,6 +664,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -667,6 +695,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -701,6 +730,7 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_banned,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
diff --git a/src/emqx_sys.erl b/src/emqx_sys.erl
index 2d816569d..65b939836 100644
--- a/src/emqx_sys.erl
+++ b/src/emqx_sys.erl
@@ -94,7 +94,17 @@ sysdescr() -> emqx_app:get_description().
 %% @doc Get sys uptime
 -spec(uptime() -> string()).
 uptime() ->
-    gen_server:call(?SYS, uptime).
+    {TotalWallClock, _} = erlang:statistics(wall_clock),
+    {D, {H, M, S}} = calendar:seconds_to_daystime(TotalWallClock div 1000),
+    List = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}],
+    {_, Uptime} =
+        lists:foldl(fun({Time, Unit}, {NeedSkip, Acc}) ->
+            case Time =:= 0 andalso NeedSkip of
+                true -> {NeedSkip, Acc};
+                false -> {false, [[integer_to_list(Time), Unit] |Acc]}
+            end
+                    end, {true, []}, List),
+    lists:flatten(lists:join(", ", lists:reverse(Uptime))).
 
 %% @doc Get sys datetime
 -spec(datetime() -> string()).
@@ -137,9 +147,6 @@ heartbeat(State) ->
 tick(State) ->
     State#state{ticker = start_timer(sys_interval(), tick)}.
 
-handle_call(uptime, _From, State) ->
-    {reply, uptime(State), State};
-
 handle_call(Req, _From, State) ->
     ?LOG(error, "Unexpected call: ~p", [Req]),
     {reply, ignored, State}.
@@ -149,7 +156,7 @@ handle_cast(Msg, State) ->
     {noreply, State}.
 
 handle_info({timeout, TRef, heartbeat}, State = #state{heartbeat = TRef}) ->
-    publish_any(uptime, iolist_to_binary(uptime(State))),
+    publish_any(uptime, iolist_to_binary(uptime())),
     publish_any(datetime, iolist_to_binary(datetime())),
     {noreply, heartbeat(State)};
 
@@ -173,24 +180,6 @@ terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) ->
 %% Internal functions
 %%-----------------------------------------------------------------------------
 
-uptime(#state{start_time = Ts}) ->
-    Secs = timer:now_diff(erlang:timestamp(), Ts) div 1000000,
-    lists:flatten(uptime(seconds, Secs)).
-uptime(seconds, Secs) when Secs < 60 ->
-    [integer_to_list(Secs), " seconds"];
-uptime(seconds, Secs) ->
-    [uptime(minutes, Secs div 60), integer_to_list(Secs rem 60), " seconds"];
-uptime(minutes, M) when M < 60 ->
-    [integer_to_list(M), " minutes, "];
-uptime(minutes, M) ->
-    [uptime(hours, M div 60), integer_to_list(M rem 60), " minutes, "];
-uptime(hours, H) when H < 24 ->
-    [integer_to_list(H), " hours, "];
-uptime(hours, H) ->
-    [uptime(days, H div 24), integer_to_list(H rem 24), " hours, "];
-uptime(days, D) ->
-    [integer_to_list(D), " days, "].
-
 publish_any(Name, Value) ->
     _ = publish(Name, Value),
     ok.

From b1816a664751b42be0370835c19c810794e3d388 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 25 Feb 2022 15:57:55 +0800
Subject: [PATCH 05/17] fix: add uptime/1 for CT test.

---
 src/emqx_sys.erl        | 18 ++++++++----------
 test/emqx_sys_SUITE.erl | 23 ++++++++++++++++++-----
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/src/emqx_sys.erl b/src/emqx_sys.erl
index 65b939836..c61272e64 100644
--- a/src/emqx_sys.erl
+++ b/src/emqx_sys.erl
@@ -95,16 +95,14 @@ sysdescr() -> emqx_app:get_description().
 -spec(uptime() -> string()).
 uptime() ->
     {TotalWallClock, _} = erlang:statistics(wall_clock),
-    {D, {H, M, S}} = calendar:seconds_to_daystime(TotalWallClock div 1000),
-    List = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}],
-    {_, Uptime} =
-        lists:foldl(fun({Time, Unit}, {NeedSkip, Acc}) ->
-            case Time =:= 0 andalso NeedSkip of
-                true -> {NeedSkip, Acc};
-                false -> {false, [[integer_to_list(Time), Unit] |Acc]}
-            end
-                    end, {true, []}, List),
-    lists:flatten(lists:join(", ", lists:reverse(Uptime))).
+    uptime(TotalWallClock div 1000).
+
+uptime(Seconds) ->
+    {D, {H, M, S}} = calendar:seconds_to_daystime(Seconds),
+    L0 = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}],
+    L1 = lists:dropwhile(fun({K, _}) ->  K =:= 0 end, L0),
+    L2 = lists:map(fun({Time, Unit}) -> [integer_to_list(Time), Unit] end, L1),
+    lists:flatten(lists:join(", ", L2)).
 
 %% @doc Get sys datetime
 -spec(datetime() -> string()).
diff --git a/test/emqx_sys_SUITE.erl b/test/emqx_sys_SUITE.erl
index 29583524f..e7289cb38 100644
--- a/test/emqx_sys_SUITE.erl
+++ b/test/emqx_sys_SUITE.erl
@@ -34,7 +34,7 @@ end_per_suite(_Config) ->
     application:unload(emqx),
     ok = emqx_logger:set_log_level(error),
     ok.
-    
+
 % t_version(_) ->
 %     error('TODO').
 
@@ -42,10 +42,23 @@ end_per_suite(_Config) ->
 %     error('TODO').
 
 t_uptime(_) ->
-    ?assertEqual(<<"1 seconds">>, iolist_to_binary(emqx_sys:uptime(seconds, 1))),
-    ?assertEqual(<<"1 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(seconds, 60))),
-    ?assertEqual(<<"1 hours, 0 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(seconds, 3600))),
-    ?assertEqual(<<"1 days, 0 hours, 0 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(seconds, 86400))).
+    ?assert(is_list(emqx_sys:uptime())),
+    ?assertEqual(<<"1 seconds">>, iolist_to_binary(emqx_sys:uptime(1))),
+    ?assertEqual(<<"1 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(60))),
+    ?assertEqual(<<"1 hours, 0 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(3600))),
+    ?assertEqual(<<"1 hours, 1 minutes, 1 seconds">>, iolist_to_binary(emqx_sys:uptime(3661))),
+    ?assertEqual(<<"1 days, 0 hours, 0 minutes, 0 seconds">>, iolist_to_binary(emqx_sys:uptime(86400))),
+    lists:map(fun({D, H, M, S}) ->
+        Expect = <<
+            (integer_to_binary(D))/binary, " days, ",
+            (integer_to_binary(H))/binary, " hours, ",
+            (integer_to_binary(M))/binary, " minutes, ",
+            (integer_to_binary(S))/binary, " seconds"
+        >>,
+        Actual = iolist_to_binary(emqx_sys:uptime(D * 86400 + H * 3600 + M * 60 + S)),
+        ?assertEqual(Expect, Actual)
+              end,
+        [{1, 2, 3, 4}, {10, 20, 30, 40}, {2222, 3, 56, 59}, {59, 23, 59, 59}]).
 
 % t_datetime(_) ->
 %     error('TODO').

From 2c47d2c3912b9f22d13b44cd6325ca91dab30d01 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 25 Feb 2022 16:26:04 +0800
Subject: [PATCH 06/17] chore: update changelog-4.3

---
 CHANGES-4.3.md | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 0e36babca..7f6d2b5a3 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -10,11 +10,6 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
-## v4.3.14
-
-### Bug fixes
-* Fix rpc get node info maybe crash when other nodes is not ready.
-
 ## v4.3.13
 
 ### Important changes
@@ -41,6 +36,7 @@ File format:
 * Fix Stomp client can not trigger `$event/client_connection` message [#7096]
 * Fix system memory false alarm at boot
 * Fix the MQTT-SN message replay when the topic is not registered to the client [#6970]
+* Fix rpc get node info maybe crash when other nodes is not ready.
 
 ## v4.3.12
 ### Important changes

From 02aa665e543909005d459cfb078985f852038710 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 11:36:56 +0100
Subject: [PATCH 07/17] chore: bump version to prepare the next release

---
 include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl
index 197ec2906..bc9c49e57 100644
--- a/include/emqx_release.hrl
+++ b/include/emqx_release.hrl
@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.12"}).
+-define(EMQX_RELEASE, {opensource, "4.3.13-rc.1"}).
 
 -else.
 

From cca9e52c9a0f475f07196e139138e137d4414118 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 11:37:13 +0100
Subject: [PATCH 08/17] chore(appup): add missing appup instructions

---
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 12 ++++++++----
 apps/emqx_rule_engine/src/emqx_rule_engine.app.src   |  2 +-
 apps/emqx_rule_engine/src/emqx_rule_engine.appup.src |  6 ++++--
 src/emqx.appup.src                                   | 10 +++++-----
 4 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
index 9ae492c5f..46b3c8312 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
@@ -13,12 +13,14 @@
      {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
    {<<"4.3.4">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
    {<<"4.3.5">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{<<"4.3.[0-3]">>,
     [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
@@ -33,10 +35,12 @@
      {delete_module,emqx_acl_mnesia_migrator},
      {delete_module,emqx_acl_mnesia_db}]},
    {<<"4.3.4">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
    {<<"4.3.5">>,
-    [{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index dd39cfc40..cd57630b4 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.7"}, % strict semver, bump manually!
+  {vsn, "4.3.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 7a758dcc4..77b41dd4a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,6 +1,7 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.6",
+  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {update,emqx_rule_metrics,{advanced,["4.3.6"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -58,7 +59,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.6",
+  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {update,emqx_rule_metrics,{advanced,["4.3.6"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 14ef02f80..c69f104e3 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,7 +1,6 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [
-    {"4.3.13",
+  [{"4.3.13",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys,brutal_purge,soft_purge,[]},
@@ -10,6 +9,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.12",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
@@ -390,9 +390,9 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [
-   {"4.3.13",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+  [{"4.3.13",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},

From 52ff180e55ef48d7971301f8c22693ebd791f48d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 16:07:42 +0100
Subject: [PATCH 09/17] chore: pin esockd 5.8.5

---
 apps/emqx_sn/rebar.config | 2 +-
 rebar.config              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config
index 5fecbb815..120d4767d 100644
--- a/apps/emqx_sn/rebar.config
+++ b/apps/emqx_sn/rebar.config
@@ -2,7 +2,7 @@
 {plugins, [rebar3_proper]}.
 
 {deps,
- [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}},
+ [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}},
   {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
  ]}.
 
diff --git a/rebar.config b/rebar.config
index 706c9bd1d..131aafebb 100644
--- a/rebar.config
+++ b/rebar.config
@@ -42,7 +42,7 @@
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
-    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.8"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}

From 0996b79d961a0bed649ad32d4fddabc54ba228ff Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 09:47:32 +0100
Subject: [PATCH 10/17] chore: add a script to simplify appup update

---
 scripts/update-appup.sh      | 61 ++++++++++++++++++++++++++++++++++++
 scripts/update_appup.escript | 55 ++++++++++++++++++++++----------
 2 files changed, 99 insertions(+), 17 deletions(-)
 create mode 100755 scripts/update-appup.sh

diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh
new file mode 100755
index 000000000..e62558a08
--- /dev/null
+++ b/scripts/update-appup.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+## This script wrapps update_appup.escript,
+## it provides a more commonly used set of default args.
+
+## Arg1: EMQX PROFILE
+
+set -euo pipefail
+
+usage() {
+    echo "$0 PROFILE PREV_VERSION"
+}
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+PROFILE="${1:-}"
+case "$PROFILE" in
+    emqx-ee)
+        DIR='enterprise'
+        TAG_PREFIX='e'
+        ;;
+    emqx)
+        DIR='broker'
+        TAG_PREFIX='v'
+        ;;
+    emqx-edge)
+        DIR='edge'
+        TAG_PREFIX='v'
+        ;;
+    *)
+        echo "Unknown profile $PROFILE"
+        usage
+        exit 1
+        ;;
+esac
+
+PREV_VERSION="$(git describe --tag --match "${TAG_PREFIX}*" | grep -oE "${TAG_PREFIX}4\.[0-9]+\.[0-9]+")"
+PREV_VERSION="${PREV_VERSION#[e|v]}"
+
+shift 1
+ESCRIPT_ARGS="$*"
+
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+ARCH="${ARCH:-$(uname -m)}"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip"
+DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}"
+
+# shellcheck disable=SC2086
+./scripts/update_appup.escript --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION"
diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 965de1efe..5bc709267 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -32,8 +32,8 @@ Options:
   --make-command    A command used to assemble the release
   --release-dir     Release directory
   --src-dirs        Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
-  --binary-rel-url  Binary release URL pattern. %TAG% variable is substituted with the release tag.
-                    E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\"
+  --binary-rel-url  Binary release URL pattern.
+                    E.g. https://www.emqx.com/downloads/broker/v4.3.12/emqx-centos7-4.3.12-amd64.zip
 ".
 
 -record(app,
@@ -171,9 +171,10 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
     BaseDir = "/tmp/emqx-baseline-bin/",
     Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
     Filename = filename:join(BaseDir, Dir),
-    Script = "mkdir -p ${OUTFILE} &&
-              wget -c -O ${OUTFILE}.zip ${URL} &&
-              unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
+    Script = "echo \"Download: ${OUTFILE}\" &&
+              mkdir -p ${OUTFILE} &&
+              curl -f -L -o ${OUTFILE}.zip ${URL} &&
+              unzip -q -n -d ${OUTFILE} ${OUTFILE}.zip",
     Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
     bash(Script, Env),
     {ok, Filename}.
@@ -208,8 +209,8 @@ find_appup_actions(App,
     {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion),
     OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0),
     OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0),
-    Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade),
-    Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade),
+    Upgrade = merge_update_actions(App, diff_app(up, App, CurrAppIdx, PrevAppIdx), OldUpgrade),
+    Downgrade = merge_update_actions(App, diff_app(down, App, PrevAppIdx, CurrAppIdx), OldDowngrade),
     if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade ->
             %% The appup file has been already updated:
             [];
@@ -521,7 +522,7 @@ index_app(AppFile) ->
               , modules       = Modules
               }}.
 
-diff_app(App,
+diff_app(UpOrDown, App,
          #app{version = NewVersion, modules = NewModules},
          #app{version = OldVersion, modules = OldModules}) ->
     {New, Changed} =
@@ -539,18 +540,31 @@ diff_app(App,
                  , NewModules
                  ),
     Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)),
-    NChanges = length(New) + length(Changed) + length(Deleted),
-    if NewVersion =:= OldVersion andalso NChanges > 0 ->
-            set_invalid(),
-            log("ERROR: Application '~p' contains changes, but its version is not updated~n", [App]);
-       NewVersion > OldVersion ->
-            log("INFO: Application '~p' has been updated: ~p -> ~p~n", [App, OldVersion, NewVersion]),
+    Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end,
+                           [{added, New}, {changed, Changed}, {deleted, Deleted}]),
+    case NewVersion =:= OldVersion of
+        true when Changes =:= [] ->
+            %% no change
             ok;
-       true ->
+        true ->
+            set_invalid(),
+            case UpOrDown =:= up of
+                true ->
+                    %% only log for the upgrade case because it would be the same result
+                    log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
+                        [App, format_changes(Changes)]);
+                false ->
+                    ok
+            end;
+        false ->
+            log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]),
             ok
     end,
     {New, Changed, Deleted}.
 
+format_changes(Changes) ->
+    lists:map(fun({Tag, List}) -> io_lib:format("~p: ~p~n", [Tag, List]) end, Changes).
+
 -spec hashsums(file:filename()) -> #{module() => binary()}.
 hashsums(EbinDir) ->
     maps:from_list(lists:map(
@@ -607,13 +621,20 @@ locate(ebin_current, App, Suffix) ->
 locate(src, App, Suffix) ->
     AppStr = atom_to_list(App),
     SrcDirs = getopt(src_dirs),
-    case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of
+    case find_app(SrcDirs ++ AppStr ++ Suffix) of
         [File] ->
             {ok, File};
         [] ->
-            undefined
+            undefined;
+        Files ->
+            error({more_than_one_app_found, Files})
     end.
 
+find_app(Pattern) ->
+    %% exclude _build dir inside apps
+    lists:filter(fun(S) -> string:find(S, "/_build/") =:= nomatch end,
+                 filelib:wildcard(Pattern)).
+
 bash(Script) ->
     bash(Script, []).
 

From ef382bf34cbf6c270a4a3ebd63c185a055fb3142 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 15:31:06 +0100
Subject: [PATCH 11/17] chore: add --make-command to update_appup script args

---
 scripts/update-appup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh
index e62558a08..e1ef0c3c0 100755
--- a/scripts/update-appup.sh
+++ b/scripts/update-appup.sh
@@ -58,4 +58,4 @@ PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip"
 DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}"
 
 # shellcheck disable=SC2086
-./scripts/update_appup.escript --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION"
+./scripts/update_appup.escript --make-command "make ${PROFILE}-rel" --binary-rel-url "$DOWNLOAD_URL" $ESCRIPT_ARGS "$PREV_VERSION"

From 802967074178a9b8a4b789b26f766644415191da Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 1 Mar 2022 14:57:17 +0100
Subject: [PATCH 12/17] ci: add appup update check to ci

---
 .github/workflows/apps_version_check.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml
index 4975e5c11..67355155f 100644
--- a/.github/workflows/apps_version_check.yaml
+++ b/.github/workflows/apps_version_check.yaml
@@ -6,7 +6,22 @@ jobs:
   check_apps_version:
     runs-on: ubuntu-20.04
 
+    strategy:
+      matrix:
+        erl_otp:
+        - erl23.2.7.2-emqx-3
+        os:
+        - ubuntu20.04
+
+    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+
     steps:
       - uses: actions/checkout@v1
       - name: Check apps version
         run: ./scripts/apps-version-check.sh
+      - name: Check relup (ce)
+        if: endsWith(github.repository, 'emqx')
+        run: ./scripts/update-appup.sh emqx --check
+      - name: Check relup (ee)
+        if: endsWith(github.repository, 'enterprise')
+        run: ./scripts/update-appup.sh emqx-ee --check

From b0900fe25a7bb262c62119cd13f78fb79d2101ba Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Mar 2022 19:36:10 +0100
Subject: [PATCH 13/17] build(update-appup): fix ARCH resolution

---
 scripts/update-appup.sh | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh
index e1ef0c3c0..db0759695 100755
--- a/scripts/update-appup.sh
+++ b/scripts/update-appup.sh
@@ -41,18 +41,20 @@ shift 1
 ESCRIPT_ARGS="$*"
 
 SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
-ARCH="${ARCH:-$(uname -m)}"
-case "$ARCH" in
-    x86_64)
-        ARCH='amd64'
-        ;;
-    aarch64)
-        ARCH='arm64'
-        ;;
-    arm*)
-        ARCH=arm
-        ;;
-esac
+if [ -z "${ARCH:-}" ]; then
+    UNAME="$(uname -m)"
+    case "$UNAME" in
+        x86_64)
+            ARCH='amd64'
+            ;;
+        aarch64)
+            ARCH='arm64'
+            ;;
+        arm*)
+            ARCH='arm'
+            ;;
+    esac
+fi
 
 PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip"
 DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}"

From 558748dab27377c4bf3d79d1f733b0579d9e9e05 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Mar 2022 19:36:30 +0100
Subject: [PATCH 14/17] build(update_appup): only download package if not exist

---
 scripts/update_appup.escript | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 5bc709267..b12012719 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -171,9 +171,11 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
     BaseDir = "/tmp/emqx-baseline-bin/",
     Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
     Filename = filename:join(BaseDir, Dir),
-    Script = "echo \"Download: ${OUTFILE}\" &&
-              mkdir -p ${OUTFILE} &&
-              curl -f -L -o ${OUTFILE}.zip ${URL} &&
+    Script = "mkdir -p ${OUTFILE} &&
+              if [ ! -f \"${OUTFILE}.zip\" ]; then \
+                echo \"Download: ${OUTFILE}\" && \
+                curl -f -L -o \"${OUTFILE}.zip\" \"${URL}\"; \
+              fi &&
               unzip -q -n -d ${OUTFILE} ${OUTFILE}.zip",
     Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
     bash(Script, Env),

From a94dc87c346c8c41a9e16cced65472650707652b Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 3 Mar 2022 11:20:50 +0800
Subject: [PATCH 15/17] chore: add missed appup instructions

---
 src/emqx.appup.src | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 53b4a9ff7..dad0ca480 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -7,6 +7,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -16,6 +17,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -40,6 +42,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@@ -49,6 +52,7 @@
      {load_module,emqx_pmon,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
      {load_module,emqx_ctl,brutal_purge,soft_purge,[]},

From c5a6b4739058a86ca077335bf395114c4365c19d Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 3 Mar 2022 11:50:59 +0800
Subject: [PATCH 16/17] build(ci): fix package name

---
 .github/workflows/apps_version_check.yaml | 4 ++--
 scripts/update-appup.sh                   | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml
index 67355155f..a1bf859ba 100644
--- a/.github/workflows/apps_version_check.yaml
+++ b/.github/workflows/apps_version_check.yaml
@@ -9,11 +9,11 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-3
+        - 24.1.5-3
         os:
         - ubuntu20.04
 
-    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v1
diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh
index db0759695..5fd26355e 100755
--- a/scripts/update-appup.sh
+++ b/scripts/update-appup.sh
@@ -40,6 +40,8 @@ PREV_VERSION="${PREV_VERSION#[e|v]}"
 shift 1
 ESCRIPT_ARGS="$*"
 
+OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}"
+
 SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
 if [ -z "${ARCH:-}" ]; then
     UNAME="$(uname -m)"
@@ -56,7 +58,7 @@ if [ -z "${ARCH:-}" ]; then
     esac
 fi
 
-PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip"
+PACKAGE_NAME="${PROFILE}-${PREV_VERSION}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
 DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}"
 
 # shellcheck disable=SC2086

From dbacfb2bb4a56cb27eb2c9d56c33ed0a91063b89 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 3 Mar 2022 22:02:27 +0800
Subject: [PATCH 17/17] test(mgmt): ensure application loaded

---
 apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl
index 1878dec45..94d354e25 100644
--- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl
+++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl
@@ -47,6 +47,7 @@ groups() ->
     ]}].
 
 init_per_suite(Config) ->
+    application:load(emqx_plugin_libs),
     emqx_ct_helpers:start_apps( [emqx_modules, emqx_management, emqx_auth_mnesia]
                               , fun set_special_configs/1
                               ),