From 49bfabc4dc37b6cd2b19c8010c8f2b1efa99112d Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Fri, 17 Dec 2021 19:41:41 +0800
Subject: [PATCH 01/46] chore(vsn): bump ehttpc and ekka version

---
 rebar.config | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rebar.config b/rebar.config
index bd21d8422..25bd95089 100644
--- a/rebar.config
+++ b/rebar.config
@@ -37,13 +37,13 @@
 
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
-    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.13"}}}
+    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.14"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.6"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.7"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}}

From dc9631514d05a87810f59710da3c14e61f309e2e Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Fri, 17 Dec 2021 20:28:54 +0800
Subject: [PATCH 02/46] chore: ensure version bump for dashboard app

---
 include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl
index e9f954b3c..e919c12de 100644
--- a/include/emqx_release.hrl
+++ b/include/emqx_release.hrl
@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.10"}).
+-define(EMQX_RELEASE, {opensource, "4.3.11"}).
 
 -else.
 

From c1cbf395525ef5483a4058521f3c1aa9397929cc Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Dec 2021 13:41:25 +0100
Subject: [PATCH 03/46] fix(appup): add missing beams to emqx.appup.src

also re-formated emqx_sn.appup.src
---
 apps/emqx_sn/src/emqx_sn.appup.src | 39 ++++++++++--------------------
 src/emqx.appup.src                 | 14 ++++++++---
 2 files changed, 23 insertions(+), 30 deletions(-)

diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src
index caf715adc..22d2cd606 100644
--- a/apps/emqx_sn/src/emqx_sn.appup.src
+++ b/apps/emqx_sn/src/emqx_sn.appup.src
@@ -1,27 +1,14 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
 {VSN,
- [
-    {"4.3.3", [
-      {load_module, emqx_sn_registry, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.2", [
-      {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []},
-      {load_module, emqx_sn_registry, brutal_purge, soft_purge, []}
-    ]},
-   {<<"4\\.3\\.[0-1]">>, [
-     {restart_application, emqx_sn}
-   ]}
- ],
- [
-   {"4.3.3", [
-      {load_module, emqx_sn_registry, brutal_purge, soft_purge, []}
-   ]},
-   {"4.3.2", [
-      {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []},
-      {load_module, emqx_sn_registry, brutal_purge, soft_purge, []}
-    ]},
-   {<<"4\\.3\\.[0-1]">>, [
-     {restart_application, emqx_sn}
-   ]}
- ]
-}.
+  [{"4.3.3",[{load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{restart_application,emqx_sn}]}],
+  [{"4.3.3",[{load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{restart_application,emqx_sn}]}]}.
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index cea35e90d..456f89e6c 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,12 +1,15 @@
 %% -*- mode: erlang -*-
 {VSN,
   [{"4.3.11",
-    [{load_module,emqx_vm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_vm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
@@ -194,12 +197,15 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.11",
-    [{load_module,emqx_vm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_vm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},

From 83734612a261037943ac8ccc49fcc5a0c15d9f6d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Dec 2021 16:05:42 +0100
Subject: [PATCH 04/46] chore: add CHANGE.md

---
 CHANGES.md | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
 create mode 100644 CHANGES.md

diff --git a/CHANGES.md b/CHANGES.md
new file mode 100644
index 000000000..d388628c1
--- /dev/null
+++ b/CHANGES.md
@@ -0,0 +1,95 @@
+# EMQ X 4.3 Changes
+
+Started tracking changes in CHANGE.md since EMQ X v4.3.11
+
+NOTE: Keep pre-pedning to the head of the file instead of the tail
+
+File format:
+
+- Use weight-2 heading for releases
+- One list item per change topic
+  Change log ends with a list of github PRs
+
+## v4.3.11
+
+Important notes:
+
+- For Debian/Ubuntu users
+
+  We changed package installed service from init.d to systemd.
+  The upgrade from init.d to systemd is verified, however it is
+  recommended to verify it before rolling out to production.
+  At least to ensure systemd is available in your system.
+
+- For Centos Users
+
+  RPM package now depends on `openssl11` which is NOT available
+  in certain centos distributions.
+  Please make sure the yum repo [epel-release](https://docs.fedoraproject.org/en-US/epel) is installed.
+
+### Important changes
+
+* Debian/Ubuntu package (deb) installed EMQ X now runs on systemd [#6389]<br>
+  This is to take advantage of systemd's supervision functionality to ensure
+  EMQ X service is restarted after crash.
+
+* Clustering malfunction fixes [#6221, #6225, #6381]
+
+### Minor changes
+
+* Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
+  "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
+
+* Helm chart supports networking.k8s.io/v1 [#6368]
+
+* Fix session takeover race condition which may lead to message loss [#6396]
+
+* EMQ X docker images are pushed to aws public ecr in an automated CI job [#6271]<br>
+  `docker pull public.ecr.aws/emqx/emqx:4.3.10`
+
+* Fix webhook URL path to allow rule-engine variable substitution [#6399]
+
+* Corrected RAM usage display [#6379]
+
+* Changed emqx_sn_registry table creation to runtime [#6357]<br>
+  This was a bug introduced in 4.3.3, in whihch the table is changed from ets to mnesia<br>
+  this will cause upgrade to fail when a later version node joins a 4.3.0-2 cluster<br>
+
+* Log level for normal termination changed from info to debug [#6358]
+
+* Added config `retainer.stop_publish_clear_msg` to enable/disable empty message retained message publish [#6343]<br>
+  In MQTT 3.1.1, it is unclear if a MQTT broker should publish the 'clear' (no payload) message<br>
+  to the subscribers, or just delete the retained message. So we have made it configurable
+
+* Fix mqtt bridge malfunction when remote host is unreachable (hangs the connection) [#6286, #6323]
+
+* System monotor now inspects `current_stacktrace` of suspicious process [#6290]<br>
+  `current_function` was not quite helpful
+
+* Changed default `max_topc_levels` config value to 128 [#6294, #6420]<br>
+  previously it has no limit (config value = 0), which can be a potential DoS threat
+
+* Collect only libcrypto and libtinfo so files for zip package [#6259]<br>
+  in 4.3.10 we tried to collect all so files, however glibc is not quite portable
+
+* Added openssl-1.1 to RPM dependency [#6239]
+
+* Http client duplicated header fix [#6195]
+
+* Fix `node_dump` issues when working with deb or rpm installation [#6209]
+
+* Pin Erlang/OTP 23.2.7.2-emqx-3 [#6246]<br>
+  4.3.10 is on 23.2.7.2-emqx-2, this bump is to fix an ECC signature name typo:
+  ecdsa_secp512r1_sha512 -> ecdsa_secp521r1_sha512
+
+* HTTP client performance improvement [#6474, #6414]<br>
+  The changes are mostly done in the dependency [repo](https://github.com/emqx/ehttpc).
+
+* For messages from gateways add message properties as MQTT message headers [#6142]<br>
+  e.g. messages from CoAP, LwM2M, Stomp, ExProto, when translated into MQTT message<br>
+  properties such as protocol name, protocol version, username (if any) peer-host<br>
+  etc. are filled as MQTT message headers.
+
+## v4.3.0~10
+
+Older version changes are not tracked here.

From caa6134e3bb60333d0cce592642180ed5fbdcb07 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Dec 2021 16:37:09 +0100
Subject: [PATCH 05/46] chore: fix CHANGES.md typos

---
 CHANGES.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index d388628c1..6956a4b7e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,7 +2,7 @@
 
 Started tracking changes in CHANGE.md since EMQ X v4.3.11
 
-NOTE: Keep pre-pedning to the head of the file instead of the tail
+NOTE: Keep prepending to the head of the file instead of the tail
 
 File format:
 
@@ -16,7 +16,7 @@ Important notes:
 
 - For Debian/Ubuntu users
 
-  We changed package installed service from init.d to systemd.
+  We changed the package installed service from init.d to systemd.
   The upgrade from init.d to systemd is verified, however it is
   recommended to verify it before rolling out to production.
   At least to ensure systemd is available in your system.
@@ -31,7 +31,7 @@ Important notes:
 
 * Debian/Ubuntu package (deb) installed EMQ X now runs on systemd [#6389]<br>
   This is to take advantage of systemd's supervision functionality to ensure
-  EMQ X service is restarted after crash.
+  EMQ X service is restarted after crashes.
 
 * Clustering malfunction fixes [#6221, #6225, #6381]
 
@@ -52,7 +52,7 @@ Important notes:
 * Corrected RAM usage display [#6379]
 
 * Changed emqx_sn_registry table creation to runtime [#6357]<br>
-  This was a bug introduced in 4.3.3, in whihch the table is changed from ets to mnesia<br>
+  This was a bug introduced in 4.3.3, in which the table is changed from ets to mnesia<br>
   this will cause upgrade to fail when a later version node joins a 4.3.0-2 cluster<br>
 
 * Log level for normal termination changed from info to debug [#6358]
@@ -63,7 +63,7 @@ Important notes:
 
 * Fix mqtt bridge malfunction when remote host is unreachable (hangs the connection) [#6286, #6323]
 
-* System monotor now inspects `current_stacktrace` of suspicious process [#6290]<br>
+* System monitor now inspects `current_stacktrace` of suspicious process [#6290]<br>
   `current_function` was not quite helpful
 
 * Changed default `max_topc_levels` config value to 128 [#6294, #6420]<br>

From ccca1f53fa333c1130f19e00da519e95694d314e Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Dec 2021 19:49:15 +0100
Subject: [PATCH 06/46] chore: add details about ekka fix

---
 CHANGES.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 6956a4b7e..a442ff6c8 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -33,10 +33,13 @@ Important notes:
   This is to take advantage of systemd's supervision functionality to ensure
   EMQ X service is restarted after crashes.
 
-* Clustering malfunction fixes [#6221, #6225, #6381]
-
 ### Minor changes
 
+* Clustering malfunction fixes [#6221, #6381]
+  Mostly changes made in [ekka](https://github.com/emqx/ekka/pull/134)<br>
+  From 0.8.1.4 to 0.8.1.6, fixes included intra-cluster RPC call timeouts,<br>
+  also fixed `ekka_locker` process crashe after killing a hanged lock owner.
+
 * Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
   "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
 

From 3ff6661a589186ebae16db2bbe12ea290c2a0ced Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 17 Dec 2021 14:48:41 -0300
Subject: [PATCH 07/46] chore(update_appup): take regexes into account when
 comparing vsns

This change makes the `update_appup.escript` check whether the new
version of an application (the _current_ one) is already contained in
entries in the _new_ .appup file for that application if such .appup
file contains regexes.

NOTE: this does not cover the case in which we calculate the
difference between _old_ and _new_ appup entries, and those consist of
regexes.  In such case, we would need to check if one regex is
"contained" in the other, which is not currently supported by this
patch.
---
 scripts/update_appup.escript | 56 +++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 16 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 7674a941c..df2f2b164 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -189,8 +189,11 @@ find_appup_actions(CurrApps, PrevApps) ->
     maps:fold(
       fun(App, CurrAppIdx, Acc) ->
               case PrevApps of
-                  #{App := PrevAppIdx} -> find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc;
-                  _                    -> Acc %% New app, nothing to upgrade here.
+                  #{App := PrevAppIdx} ->
+                      find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc;
+                  _ ->
+                      %% New app, nothing to upgrade here.
+                      Acc
               end
       end,
       [],
@@ -214,10 +217,10 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
 %% in their current appup.
 diff_appup_instructions(ComputedChanges, PresentChanges) ->
     lists:foldr(
-      fun({Vsn, ComputedActions}, Acc) ->
-              case find_matching_version(Vsn, PresentChanges) of
+      fun({VsnOrRegex, ComputedActions}, Acc) ->
+              case find_matching_version(VsnOrRegex, PresentChanges) of
                   undefined ->
-                      [{Vsn, ComputedActions} | Acc];
+                      [{VsnOrRegex, ComputedActions} | Acc];
                   PresentActions ->
                       DiffActions = ComputedActions -- PresentActions,
                       case DiffActions of
@@ -225,7 +228,7 @@ diff_appup_instructions(ComputedChanges, PresentChanges) ->
                               %% no diff
                               Acc;
                           _ ->
-                              [{Vsn, DiffActions} | Acc]
+                              [{VsnOrRegex, DiffActions} | Acc]
                       end
               end
       end,
@@ -250,8 +253,11 @@ parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) ->
     end.
 
 %% TODO: handle regexes
-find_matching_version(Vsn, PresentChanges) ->
-    proplists:get_value(Vsn, PresentChanges).
+%% Since the first argument may be a regex itself, we would need to
+%% check if it is "contained" within other regexes inside list of
+%% versions in the second argument.
+find_matching_version(VsnOrRegex, PresentChanges) ->
+    proplists:get_value(VsnOrRegex, PresentChanges).
 
 find_old_appup_actions(App, PrevVersion) ->
     {Upgrade0, Downgrade0} =
@@ -279,11 +285,11 @@ merge_update_actions(App, Changes, Vsns) ->
     lists:map(fun(Ret = {<<".*">>, _}) ->
                       Ret;
                  ({Vsn, Actions}) ->
-                      {Vsn, do_merge_update_actions(App, Changes, Actions)}
+                      {Vsn, do_merge_update_actions(App, Vsn, Changes, Actions)}
               end,
               Vsns).
 
-do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) ->
+do_merge_update_actions(App, Vsn, {New0, Changed0, Deleted0}, OldActions) ->
     AppSpecific = app_specific_actions(App) -- OldActions,
     AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)),
     New = New0 -- AlreadyHandled,
@@ -308,14 +314,30 @@ process_old_action(_) ->
     [].
 
 ensure_version(Version, OldInstructions) ->
-    OldVersions = [ensure_string(element(1, I)) || I <- OldInstructions],
-    case lists:member(Version, OldVersions) of
+    OldVersions = [element(1, I) || I <- OldInstructions],
+    case contains_version(Version, OldVersions) of
         false ->
-            [{Version, []}|OldInstructions];
+            [{Version, []} | OldInstructions];
         _ ->
             OldInstructions
     end.
 
+contains_version(Needle, Haystack) when is_list(Needle) ->
+    lists:any(
+      fun(Regex) when is_binary(Regex) ->
+              case re:run(Needle, Regex) of
+                  {match, _} ->
+                      true;
+                  nomatch ->
+                      false
+              end;
+         (Needle) ->
+              true;
+         (_) ->
+              false
+      end,
+      Haystack).
+
 read_appup(File) ->
     %% NOTE: appup file is a script, it may contain variables or functions.
     case file:script(File, [{'VSN', "VSN"}]) of
@@ -398,16 +420,18 @@ index_app(AppFile) ->
               , modules       = Modules
               }}.
 
-diff_app(App, #app{version = NewVersion, modules = NewModules}, #app{version = OldVersion, modules = OldModules}) ->
+diff_app(App,
+         #app{version = NewVersion, modules = NewModules},
+         #app{version = OldVersion, modules = OldModules}) ->
     {New, Changed} =
         maps:fold( fun(Mod, MD5, {New, Changed}) ->
                            case OldModules of
                                #{Mod := OldMD5} when MD5 =:= OldMD5 ->
                                    {New, Changed};
                                #{Mod := _} ->
-                                   {New, [Mod|Changed]};
+                                   {New, [Mod | Changed]};
                                _ ->
-                                   {[Mod|New], Changed}
+                                   {[Mod | New], Changed}
                            end
                    end
                  , {[], []}

From e1e72c144a696dde51b475b3e7978ddb324ccbc7 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 17 Dec 2021 15:50:47 -0300
Subject: [PATCH 08/46] chore(update_appup): do not use load_module if
 restart_application

Since the appup instruction `restart_application` already loads all
modules of a given application, there is no need to introduce those
instructions if a restart is already present.
---
 scripts/update_appup.escript | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index df2f2b164..65be8239b 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -295,11 +295,18 @@ do_merge_update_actions(App, Vsn, {New0, Changed0, Deleted0}, OldActions) ->
     New = New0 -- AlreadyHandled,
     Changed = Changed0 -- AlreadyHandled,
     Deleted = Deleted0 -- AlreadyHandled,
-    [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++
+    Reloads = [{load_module, M, brutal_purge, soft_purge, []}
+               || not contains_restart_application(App, OldActions),
+                  M <- Changed ++ New],
+    Reloads ++
         OldActions ++
         [{delete_module, M} || M <- Deleted] ++
         AppSpecific.
 
+%% If an entry restarts an application, there's no need to use
+%% `load_module' instructions.
+contains_restart_application(Application, Actions) ->
+    lists:member({restart_application, Application}, Actions).
 
 %% @doc Process the existing actions to exclude modules that are
 %% already handled

From 42ca5ab5a974f4fa99195a6585c75834f158cb3f Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 17 Dec 2021 16:01:48 -0300
Subject: [PATCH 09/46] chore(update_appup): do not force appup render if
 contents are the same

To avoid losing comments and/or manual indentation in appup files that
are already up to date, we now check whether the contents have the
exact same terms as those we are about to write to an existint .appup
file.
---
 scripts/update_appup.escript | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 65be8239b..530f15072 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -368,7 +368,12 @@ update_appups(Changes) ->
 do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
     case locate(src, App, ".appup.src") of
         {ok, AppupFile} ->
-            render_appfile(AppupFile, Upgrade, Downgrade);
+            case contains_contents(AppupFile, Upgrade, Downgrade) of
+                true ->
+                    ok;
+                false ->
+                    render_appfile(AppupFile, Upgrade, Downgrade)
+            end;
         undefined ->
             case create_stub(App) of
                 {ok, AppupFile} ->
@@ -408,6 +413,17 @@ create_stub(App) ->
             false
     end.
 
+%% we check whether the destination file already has the contents we
+%% want to write to avoid writing and losing indentation and comments.
+contains_contents(File, Upgrade, Downgrade) ->
+    %% the file may contain the VSN variable, so it's a script
+    case file:script(File, [{'VSN', 'VSN'}]) of
+        {ok, {_, Upgrade, Downgrade}} ->
+            true;
+        _ ->
+            false
+    end.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% application and release indexing
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From fef8a18bfb8c3a09395d91fdab97985ed6cd97e6 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 17 Dec 2021 16:31:04 -0300
Subject: [PATCH 10/46] chore(update_appup): insert `load_module`s after
 `application:stop`

If there is already any `application:stop(Application)` call in the
appup instructions, we prefer to add `load_module` instructions after
it, so we can be sure that the load is replaced safely.
---
 scripts/update_appup.escript | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 530f15072..efd98b046 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -298,8 +298,11 @@ do_merge_update_actions(App, Vsn, {New0, Changed0, Deleted0}, OldActions) ->
     Reloads = [{load_module, M, brutal_purge, soft_purge, []}
                || not contains_restart_application(App, OldActions),
                   M <- Changed ++ New],
-    Reloads ++
-        OldActions ++
+    {OldActionsWithStop, OldActionsAfterStop} =
+        find_application_stop_instruction(App, OldActions),
+    OldActionsWithStop ++
+        Reloads ++
+        OldActionsAfterStop ++
         [{delete_module, M} || M <- Deleted] ++
         AppSpecific.
 
@@ -308,6 +311,23 @@ do_merge_update_actions(App, Vsn, {New0, Changed0, Deleted0}, OldActions) ->
 contains_restart_application(Application, Actions) ->
     lists:member({restart_application, Application}, Actions).
 
+%% If there is an `application:stop(Application)' call in the
+%% instructions, we insert `load_module' instructions after it.
+find_application_stop_instruction(Application, Actions) ->
+    {Before, After0} =
+        lists:splitwith(
+          fun({apply, {application, stop, [Application]}}) ->
+                  false;
+             (_) ->
+                  true
+          end, Actions),
+    case After0 of
+        [StopInst | After] ->
+            {Before ++ [StopInst], After};
+        [] ->
+            {[], Before}
+    end.
+
 %% @doc Process the existing actions to exclude modules that are
 %% already handled
 process_old_action({purge, Modules}) ->

From f78be5a4be649347b42b4ae9b8d9f37879d2fa2b Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Mon, 20 Dec 2021 10:25:23 +0800
Subject: [PATCH 11/46] ci(release): only push emqx broker image to aws ecr

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 1aafd3cc6..aa93c8ba5 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -453,7 +453,7 @@ jobs:
              -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
              ${{ secrets.EMQX_IO_RELEASE_API }}
     - name: push docker image to docker hub
-      if: github.event_name == 'release'
+      if: github.event_name == 'release' && matrix.profile == 'emqx'
       run: |
         set -e -x -u
         sudo make docker-prepare

From 22814dc1617fabf99968adf710f0319d8192f7cb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 20 Dec 2021 15:07:29 +0800
Subject: [PATCH 12/46] chore(coap): update gen_coap to v0.4.2

---
 apps/emqx_coap/rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config
index 0f8759b8a..e1da2c197 100644
--- a/apps/emqx_coap/rebar.config
+++ b/apps/emqx_coap/rebar.config
@@ -1,4 +1,4 @@
 {deps,
  [
-  {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.2"}}}
+  {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.4.2"}}}
  ]}.

From 23e4e101a4ef6365633a0fd46806c310a6bfa91b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 20 Dec 2021 15:42:16 +0800
Subject: [PATCH 13/46] chore(dashboard): bump lib-ce/emqx_dashboard to 4.3.8

---
 lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
index e78f1c3f6..ea5ecdd79 100644
--- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
@@ -1,6 +1,6 @@
 {application, emqx_dashboard,
  [{description, "EMQ X Web Dashboard"},
-  {vsn, "4.3.7"}, % strict semver, bump manually!
+  {vsn, "4.3.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_dashboard_sup]},
   {applications, [kernel,stdlib,mnesia,minirest]},

From af3a1326d13c972da79c7a1f67e2cac832599b41 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 20 Dec 2021 13:07:28 -0300
Subject: [PATCH 14/46] chore(update_appup): bugfix: variable not pinned in
 lambda

---
 scripts/update_appup.escript | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index efd98b046..050a180bb 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -316,7 +316,7 @@ contains_restart_application(Application, Actions) ->
 find_application_stop_instruction(Application, Actions) ->
     {Before, After0} =
         lists:splitwith(
-          fun({apply, {application, stop, [Application]}}) ->
+          fun({apply, {application, stop, [App]}}) when App =:= Application ->
                   false;
              (_) ->
                   true
@@ -345,7 +345,7 @@ ensure_version(Version, OldInstructions) ->
     case contains_version(Version, OldVersions) of
         false ->
             [{Version, []} | OldInstructions];
-        _ ->
+        true ->
             OldInstructions
     end.
 
@@ -358,10 +358,8 @@ contains_version(Needle, Haystack) when is_list(Needle) ->
                   nomatch ->
                       false
               end;
-         (Needle) ->
-              true;
-         (_) ->
-              false
+         (Vsn) ->
+              Vsn =:= Needle
       end,
       Haystack).
 

From ce4a193cbb8009b417f3f9fbbec983da475f671d Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 20 Dec 2021 13:08:01 -0300
Subject: [PATCH 15/46] chore(update_appup): rm unused fn

---
 scripts/update_appup.escript | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 050a180bb..c57272085 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -587,10 +587,5 @@ log(Msg) ->
 log(Msg, Args) ->
     io:format(standard_error, Msg, Args).
 
-ensure_string(Str) when is_binary(Str) ->
-    binary_to_list(Str);
-ensure_string(Str) when is_list(Str) ->
-    Str.
-
 otp_standard_apps() ->
     [ssl, mnesia, kernel, asn1, stdlib].

From b2396438a0efcadfc652e0604e114b6ccc54fb7a Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 20 Dec 2021 13:38:08 -0300
Subject: [PATCH 16/46] chore(update_appup): add expected versions check

For apps inside emqx umbrella, we try to bump only the patch part of
their version numbers, and use only 3-part version
numbers (`Major.Minor.Patch`).  With those assumptions, we may infer
all versions that need to be covered in a given upgrade, and check if
those are covered in regexes.
---
 scripts/update_appup.escript | 75 ++++++++++++++++++++++++++++++++++--
 1 file changed, 72 insertions(+), 3 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index c57272085..6dfbc4c1b 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -202,8 +202,12 @@ find_appup_actions(CurrApps, PrevApps) ->
 find_appup_actions(_App, AppIdx, AppIdx) ->
     %% No changes to the app, ignore:
     [];
-find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
-    {OldUpgrade, OldDowngrade} = find_old_appup_actions(App, PrevVersion),
+find_appup_actions(App,
+                   CurrAppIdx = #app{version = CurrVersion},
+                   PrevAppIdx = #app{version = PrevVersion}) ->
+    {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion),
+    OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0),
+    OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0),
     Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade),
     Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade),
     if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade ->
@@ -213,6 +217,32 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
             [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}]
     end.
 
+%% To avoid missing one patch version when upgrading, we try to
+%% optimistically generate the list of expected versions that should
+%% be covered by the upgrade.
+ensure_all_patch_versions(App, CurrVsn, OldActions) ->
+    case is_app_external(App) of
+        true ->
+            %% we do not attempt to predict the version list for
+            %% external dependencies, as those may not follow our
+            %% conventions.
+            OldActions;
+        false ->
+            do_ensure_all_patch_versions(App, CurrVsn, OldActions)
+    end.
+
+do_ensure_all_patch_versions(App, CurrVsn, OldActions) ->
+    case enumerate_past_versions(CurrVsn) of
+        {ok, ExpectedVsns} ->
+            CoveredVsns = [V || {V, _} <- OldActions, V =/= <<".*">>],
+            ExpectedVsnStrs = [vsn_number_to_string(V) || V <- ExpectedVsns],
+            MissingActions = [{V, []} || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns)],
+            MissingActions ++ OldActions;
+        {error, bad_version} ->
+            log("WARN: Could not infer expected versions to upgrade from for ~p~n", [App]),
+            OldActions
+    end.
+
 %% For external dependencies, show only the changes that are missing
 %% in their current appup.
 diff_appup_instructions(ComputedChanges, PresentChanges) ->
@@ -363,6 +393,35 @@ contains_version(Needle, Haystack) when is_list(Needle) ->
       end,
       Haystack).
 
+%% As a best effort approach, we assume that we only bump patch
+%% version numbers between release upgrades for our dependencies and
+%% that we deal only with 3-part version schemas
+%% (`Major.Minor.Patch').  Using those assumptions, we enumerate the
+%% past versions that should be covered by regexes in .appup file
+%% instructions.
+enumerate_past_versions(Vsn) when is_list(Vsn) ->
+    case parse_version_number(Vsn) of
+        {ok, ParsedVsn} ->
+            {ok, enumerate_past_versions(ParsedVsn)};
+        Error ->
+            Error
+    end;
+enumerate_past_versions({Major, Minor, Patch}) ->
+    [{Major, Minor, P} || P <- lists:seq(Patch - 1, 0, -1)].
+
+parse_version_number(Vsn) when is_list(Vsn) ->
+    Nums = string:split(Vsn, ".", all),
+    Results = lists:map(fun string:to_integer/1, Nums),
+    case Results of
+        [{Major, []}, {Minor, []}, {Patch, []}] ->
+            {ok, {Major, Minor, Patch}};
+        _ ->
+            {error, bad_version}
+    end.
+
+vsn_number_to_string({Major, Minor, Patch}) ->
+    io_lib:format("~b.~b.~b", [Major, Minor, Patch]).
+
 read_appup(File) ->
     %% NOTE: appup file is a script, it may contain variables or functions.
     case file:script(File, [{'VSN', "VSN"}]) of
@@ -419,7 +478,8 @@ render_appfile(File, Upgrade, Downgrade) ->
     ok = file:write_file(File, IOList).
 
 create_stub(App) ->
-    case locate(src, App, Ext = ".app.src") of
+    Ext = ".app.src",
+    case locate(src, App, Ext) of
         {ok, AppSrc} ->
             DirName = filename:dirname(AppSrc),
             AppupFile = filename:basename(AppSrc, Ext) ++ ".appup.src",
@@ -502,6 +562,15 @@ hashsums(EbinDir) ->
                      filelib:wildcard("*.beam", EbinDir)
                     )).
 
+is_app_external(App) ->
+    Ext = ".app.src",
+    case locate(src, App, Ext) of
+        {ok, _} ->
+            false;
+        undefined ->
+            true
+    end.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Global state
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From 62ff6a8b30ff90d8a33f9c890c5cdcadeed4bb93 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 20 Dec 2021 13:41:14 -0300
Subject: [PATCH 17/46] chore(update_appup): rm unused value

---
 scripts/update_appup.escript | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index 6dfbc4c1b..965de1efe 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -315,11 +315,11 @@ merge_update_actions(App, Changes, Vsns) ->
     lists:map(fun(Ret = {<<".*">>, _}) ->
                       Ret;
                  ({Vsn, Actions}) ->
-                      {Vsn, do_merge_update_actions(App, Vsn, Changes, Actions)}
+                      {Vsn, do_merge_update_actions(App, Changes, Actions)}
               end,
               Vsns).
 
-do_merge_update_actions(App, Vsn, {New0, Changed0, Deleted0}, OldActions) ->
+do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) ->
     AppSpecific = app_specific_actions(App) -- OldActions,
     AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)),
     New = New0 -- AlreadyHandled,

From 0ee589ca612ed3994ae58cac2271bc95aff5ab32 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 21 Dec 2021 01:07:01 -0500
Subject: [PATCH 18/46] fix(mgmt): fix node info format for specific node

---
 apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
index 2be151aaa..2266ccfa3 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
@@ -36,11 +36,10 @@ list(_Bindings, _Params) ->
     minirest:return({ok, [format(Node, Info) || {Node, Info} <- emqx_mgmt:list_nodes()]}).
 
 get(#{node := Node}, _Params) ->
-    minirest:return({ok, emqx_mgmt:lookup_node(Node)}).
+    minirest:return({ok, format(Node, emqx_mgmt:lookup_node(Node))}).
 
 format(Node, {error, Reason}) -> #{node => Node, error => Reason};
 
 format(_Node, Info = #{memory_total := Total, memory_used := Used}) ->
     Info#{memory_total := emqx_mgmt_util:kmg(Total),
           memory_used  := emqx_mgmt_util:kmg(Used)}.
-

From 7e9870cc3f973bd140f01b72503520234732fa10 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 21 Dec 2021 01:11:57 -0500
Subject: [PATCH 19/46] chore(mgmt): update appup.src

---
 apps/emqx_management/src/emqx_management.app.src   | 2 +-
 apps/emqx_management/src/emqx_management.appup.src | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index 23f58c94a..e8b235be7 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.9"}, % strict semver, bump manually!
+  {vsn, "4.3.10"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},
diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src
index f0478add1..1463334b4 100644
--- a/apps/emqx_management/src/emqx_management.appup.src
+++ b/apps/emqx_management/src/emqx_management.appup.src
@@ -1,13 +1,13 @@
 %% -*- mode: erlang -*-
 {VSN,
- [ {<<"4\\.3\\.[0-8]+">>,
+ [ {<<"4\\.3\\.[0-9]+">>,
     [ {apply,{minirest,stop_http,['http:management']}},
       {apply,{minirest,stop_http,['https:management']}},
       {restart_application, emqx_management}
     ]},
    {<<".*">>, []}
  ],
- [ {<<"4\\.3\\.[0-8]+">>,
+ [ {<<"4\\.3\\.[0-9]+">>,
     [ {apply,{minirest,stop_http,['http:management']}},
       {apply,{minirest,stop_http,['https:management']}},
       {restart_application, emqx_management}

From a560174ad9da1df3ce642f0f969e9105ad3162ad Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Wed, 22 Dec 2021 09:40:09 +0800
Subject: [PATCH 20/46] ci(cts): use makefile target instead of the rebar
 command

---
 .github/workflows/run_cts_tests.yaml | 20 +++++---------------
 Makefile                             |  2 +-
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml
index 53dd74b6a..fd074a18f 100644
--- a/.github/workflows/run_cts_tests.yaml
+++ b/.github/workflows/run_cts_tests.yaml
@@ -48,9 +48,7 @@ jobs:
         run: |
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
+          docker exec --env-file .env -i erlang sh -c ".make apps/emqx_auth_ldap-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -117,9 +115,7 @@ jobs:
         run: |
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
+          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mongo-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -199,9 +195,7 @@ jobs:
         run: |
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
+          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mysql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -273,9 +267,7 @@ jobs:
                  EMQX_AUTH__PGSQL__DATABASE=mqtt \
                  CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
+          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_pgsql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -394,9 +386,7 @@ jobs:
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           export EMQX_AUTH__REIDS__PASSWORD=public
           printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
+          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_redis-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
diff --git a/Makefile b/Makefile
index c352089d6..176d7c145 100644
--- a/Makefile
+++ b/Makefile
@@ -51,7 +51,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct:
+$1-ct: $(REBAR)
 	$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))

From 5fbd999ef8261d0dbea6600f5435a60ab542fb8b Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Wed, 22 Dec 2021 16:01:58 +0800
Subject: [PATCH 21/46] ci(cts): fix env error

---
 .github/workflows/run_cts_tests.yaml | 44 ++++++++++++++++++----------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml
index fd074a18f..84aa301c4 100644
--- a/.github/workflows/run_cts_tests.yaml
+++ b/.github/workflows/run_cts_tests.yaml
@@ -3,6 +3,7 @@ name: Compatibility Test Suite
 on:
   schedule:
     - cron:  '0 */6 * * *'
+  pull_request:
   push:
     tags:
       - v*
@@ -46,9 +47,12 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec --env-file .env -i erlang sh -c ".make apps/emqx_auth_ldap-ct"
+          docker exec -i erlang sh -c "make ensure-rebar3"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_ldap-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -113,9 +117,11 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mongo-ct"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_mongo-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -193,9 +199,11 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mysql-ct"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_mysql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -264,10 +272,12 @@ jobs:
         run: |
           export EMQX_AUTH__PGSQL__USERNAME=root \
                  EMQX_AUTH__PGSQL__PASSWORD=public \
-                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
-                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_pgsql-ct"
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_pgsql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -383,10 +393,12 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           export EMQX_AUTH__REIDS__PASSWORD=public
-          printenv > .env
-          docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_redis-ct"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_redis-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

From cb8a3725b4cd1cef2b8091f252dba711d69c3429 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 20 Dec 2021 14:22:06 +0800
Subject: [PATCH 22/46] fix: disconnect the client due to exceed
 receive-maximum packets

As described in the 5.0 specification, we should disconnect clients that
exceed the receive-maximum limit.

> If it receives more than Receive Maximum QoS 1 and QoS 2 PUBLISH packets
where it has not sent a PUBACK or PUBCOMP in response, **the Server uses a
DISCONNECT packet with Reason Code 0x9**

fix: #6447
---
 src/emqx_channel.erl        | 2 +-
 test/emqx_channel_SUITE.erl | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl
index f33b3b337..2774ac264 100644
--- a/src/emqx_channel.erl
+++ b/src/emqx_channel.erl
@@ -599,7 +599,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
             ?LOG(warning, "Dropped the qos2 packet ~w "
                  "due to awaiting_rel is full.", [PacketId]),
             ok = emqx_metrics:inc('packets.publish.dropped'),
-            handle_out(pubrec, {PacketId, RC}, Channel)
+            handle_out(disconnect, RC, Channel)
     end.
 
 ensure_quota(_, Channel = #channel{quota = undefined}) ->
diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl
index 3249eb991..97b77ca88 100644
--- a/test/emqx_channel_SUITE.erl
+++ b/test/emqx_channel_SUITE.erl
@@ -209,7 +209,8 @@ t_handle_in_qos2_publish_with_error_return(_) ->
     {ok, ?PUBREC_PACKET(2, ?RC_NO_MATCHING_SUBSCRIBERS), Channel1} =
         emqx_channel:handle_in(Publish2, Channel),
     Publish3 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 3, <<"payload">>),
-    {ok, ?PUBREC_PACKET(3, ?RC_RECEIVE_MAXIMUM_EXCEEDED), Channel1} =
+    {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_RECEIVE_MAXIMUM_EXCEEDED)},
+          {close, receive_maximum_exceeded}], Channel1} =
         emqx_channel:handle_in(Publish3, Channel1).
 
 t_handle_in_puback_ok(_) ->

From 02dc216173b024eb616ac83fa7d80167cdb3f027 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 20 Dec 2021 14:34:40 +0800
Subject: [PATCH 23/46] chore: update appup.src

---
 src/emqx.app.src   | 2 +-
 src/emqx.appup.src | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/emqx.app.src b/src/emqx.app.src
index b0e1664cc..1119deccb 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -1,7 +1,7 @@
 {application, emqx,
  [{id, "emqx"},
   {description, "EMQ X"},
-  {vsn, "4.3.12"}, % strict semver, bump manually!
+  {vsn, "4.3.13"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]},
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 456f89e6c..c4885b7b1 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,6 +1,8 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.11",
+  [{"4.3.12",
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
+   {"4.3.11",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
@@ -196,7 +198,9 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.11",
+  [{"4.3.12",
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
+   {"4.3.11",
     [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},

From 967dc7d55eeb69ef6e000951c8151b4b2d3520dd Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Dec 2021 15:40:25 +0100
Subject: [PATCH 24/46] fix: portable shebang

---
 .../test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh   | 4 +++-
 bin/emqx                                                      | 2 +-
 bin/emqx_ctl                                                  | 2 +-
 build                                                         | 2 +-
 deploy/docker/docker-entrypoint.sh                            | 2 +-
 scripts/apps-version-check.sh                                 | 2 +-
 scripts/get-distro.sh                                         | 2 +-
 scripts/one-more-emqx-ee.sh                                   | 2 +-
 scripts/one-more-emqx.sh                                      | 2 +-
 9 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh
index e31729784..d8cce6697 100755
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh
@@ -1,5 +1,7 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
 set -eux pipefail
+
 # Helper script for creating data export files
 
 container() {
diff --git a/bin/emqx b/bin/emqx
index 6a80f4d8d..8fb7adee7 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
diff --git a/bin/emqx_ctl b/bin/emqx_ctl
index 4c3aaa1ca..8d729083d 100755
--- a/bin/emqx_ctl
+++ b/bin/emqx_ctl
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
diff --git a/build b/build
index 984c7032a..d941aa40a 100755
--- a/build
+++ b/build
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script helps to build release artifacts.
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh
index 16b6cb077..0776f957b 100755
--- a/deploy/docker/docker-entrypoint.sh
+++ b/deploy/docker/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## EMQ docker image start script
 # Huang Rui <vowstar@gmail.com>
 # EMQ X Team <support@emqx.io>
diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 596f7404a..525c73d83 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 latest_release=$(git describe --abbrev=0 --tags)
diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh
index ae52abba3..2a2d39182 100755
--- a/scripts/get-distro.sh
+++ b/scripts/get-distro.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ## This script prints Linux distro name and its version number
 ## e.g. macos, centos8, ubuntu20.04
diff --git a/scripts/one-more-emqx-ee.sh b/scripts/one-more-emqx-ee.sh
index f94681056..1b37ac5cb 100644
--- a/scripts/one-more-emqx-ee.sh
+++ b/scripts/one-more-emqx-ee.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # shellcheck disable=2090
 ###############
 ## args and env validation
diff --git a/scripts/one-more-emqx.sh b/scripts/one-more-emqx.sh
index d905f64c4..c6d362dbe 100644
--- a/scripts/one-more-emqx.sh
+++ b/scripts/one-more-emqx.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # shellcheck disable=2090
 ###############
 ## args and env validation

From 5addf9187353394700b6cebeb3ae500cdb282275 Mon Sep 17 00:00:00 2001
From: k32 <10274441+k32@users.noreply.github.com>
Date: Wed, 29 Dec 2021 16:23:24 +0100
Subject: [PATCH 25/46] fix(retainer): Fix topic printing

Fixes: #6559
---
 apps/emqx_retainer/src/emqx_retainer.app.src  |  2 +-
 .../emqx_retainer/src/emqx_retainer.appup.src | 27 +++++++++----------
 apps/emqx_retainer/src/emqx_retainer_cli.erl  |  5 ++--
 .../test/emqx_retainer_cli_SUITE.erl          | 24 +++++++++++++----
 4 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src
index c5ca7599d..05920e985 100644
--- a/apps/emqx_retainer/src/emqx_retainer.app.src
+++ b/apps/emqx_retainer/src/emqx_retainer.app.src
@@ -1,6 +1,6 @@
 {application, emqx_retainer,
  [{description, "EMQ X Retainer"},
-  {vsn, "4.3.2"}, % strict semver, bump manually!
+  {vsn, "4.3.3"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_retainer_sup]},
   {applications, [kernel,stdlib]},
diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src
index 19e8e835f..45ec6420c 100644
--- a/apps/emqx_retainer/src/emqx_retainer.appup.src
+++ b/apps/emqx_retainer/src/emqx_retainer.appup.src
@@ -1,15 +1,14 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
 {VSN,
-  [
-    {<<"4\\.3\\.[0-1]+">>, [
-      {load_module, emqx_retainer, brutal_purge, soft_purge, []}
-    ]},
-    {<<".*">>, []}
-  ],
-  [
-    {<<"4\\.3\\.[0-1]+">>, [
-      {load_module, emqx_retainer, brutal_purge, soft_purge, []}
-    ]},
-    {<<".*">>, []}
-  ]
-}.
+  [{"4.3.2",
+    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.2",
+    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.
diff --git a/apps/emqx_retainer/src/emqx_retainer_cli.erl b/apps/emqx_retainer/src/emqx_retainer_cli.erl
index fe8fa9578..23fb98252 100644
--- a/apps/emqx_retainer/src/emqx_retainer_cli.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_cli.erl
@@ -33,7 +33,9 @@ cmd(["info"]) ->
 cmd(["topics"]) ->
     case mnesia:dirty_all_keys(?TAB) of
         []     -> ignore;
-        Topics -> lists:foreach(fun(Topic) -> emqx_ctl:print("~s~n", [Topic]) end, Topics)
+        Topics -> lists:foreach(fun(Topic) ->
+                                        emqx_ctl:print("~s~n", [emqx_topic:join(Topic)])
+                                end, Topics)
     end;
 
 cmd(["clean"]) ->
@@ -55,4 +57,3 @@ cmd(_) ->
 
 unload() ->
     emqx_ctl:unregister_command(retainer).
-
diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
index f9f657483..2e60e9ac4 100644
--- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
+++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
@@ -23,18 +23,32 @@
 
 all() -> emqx_ct:all(?MODULE).
 
-init_per_testcase(_TestCase, Config) ->
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_retainer]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_ct_helpers:stop_apps([emqx_retainer]).
+
+init_per_testcase(TestCase, Config) ->
     Config.
 
 end_per_testcase(_TestCase, Config) ->
-    Config.
+    emqx_retainer:clean(<<"#">>).
 
-% t_cmd(_) ->
-%     error('TODO').
+t_cmd(_) ->
+    {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C1),
+    emqtt:publish(C1, <<"/retained">>, <<"this is a retained message">>, [{qos, 0}, {retain, true}]),
+    emqtt:publish(C1, <<"/retained/2">>, <<"this is a retained message">>, [{qos, 0}, {retain, true}]),
+    timer:sleep(1000),
+    ?assertMatch(ok, emqx_retainer_cli:cmd(["topics"])),
+    ?assertMatch(ok, emqx_retainer_cli:cmd(["info"])),
+    ?assertMatch(ok, emqx_retainer_cli:cmd(["clean", "retained"])),
+    ?assertMatch(ok, emqx_retainer_cli:cmd(["clean"])).
 
 % t_unload(_) ->
 %     error('TODO').
 
 % t_load(_) ->
 %     error('TODO').
-

From 9662eebae8c130daad19730ac455dbae1234b987 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 10:08:02 +0800
Subject: [PATCH 26/46] chore(helm): fix spell errors

---
 deploy/charts/emqx/templates/StatefulSet.yaml | 15 +++++++++++++--
 deploy/charts/emqx/values.yaml                |  4 ++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index dcc03b996..3c2ff54f2 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -82,7 +82,12 @@ spec:
           claimName: {{ tpl . $ }}
         {{- end }}
       {{- end }}
-      {{- if .Values.emqxLicneseSecretName  }}
+      {{- if .Values.emqxLicenseSecretName  }}
+      - name: emqx-license
+        secret:
+          secretName: {{ .Values.emqxLicenseSecretName }}
+      ## Compatible with previous misspellings
+      {{- else if .Values.emqxLicneseSecretName }}
       - name: emqx-license
         secret:
           secretName: {{ .Values.emqxLicneseSecretName }}
@@ -159,7 +164,13 @@ spec:
           - name: emqx-loaded-modules
             mountPath: "/opt/emqx/data/loaded_modules"
             subPath: "loaded_modules"
-          {{ if .Values.emqxLicneseSecretName  }}
+          {{ if .Values.emqxLicenseSecretName  }}
+          - name: emqx-license
+            mountPath: "/opt/emqx/etc/emqx.lic"
+            subPath: "emqx.lic"
+            readOnly: true
+          ## Compatible with previous misspellings
+          {{ else if .Values.emqxLicneseSecretName  }}
           - name: emqx-license
             mountPath: "/opt/emqx/etc/emqx.lic"
             subPath: "emqx.lic"
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index c30e237b7..4b1526cc9 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -99,10 +99,10 @@ emqxLoadedModules: >
   {emqx_mod_subscription, false}.
   {emqx_mod_topic_metrics, false}.
 
-## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicneseSecretName"
+## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName"
 ## Example:
 ##   kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic
-emqxLicneseSecretName:
+emqxLicenseSecretName:
 
 service:
   ## Service type

From 90a2ebfb0275bcb69d522a3c73341b890e901a3d Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 11:16:35 +0800
Subject: [PATCH 27/46] build(docker): update docker build

---
 Makefile                 | 16 +++++++++++++---
 build                    | 17 +++++++++++++++++
 deploy/docker/Dockerfile |  6 +-----
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 176d7c145..4fa86eda3 100644
--- a/Makefile
+++ b/Makefile
@@ -3,9 +3,12 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
+export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
 export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3
+export DOCKERFILE := deploy/docker/Dockerfile
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 endif
@@ -148,11 +151,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
-	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk
+	./_build/$(PROFILE)/rel/emqx/bin/emqx console
\ No newline at end of file
diff --git a/build b/build
index d941aa40a..46cd60bc9 100755
--- a/build
+++ b/build
@@ -128,6 +128,19 @@ make_zip() {
     (cd "${tard}" && zip -qr - emqx) > "${zipball}"
 }
 
+## This function builds the default docker image based on alpine:3.14 (by default)
+make_docker() {
+    EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
+    EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
+    set -x
+    docker build --no-cache --pull \
+       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
+       --build-arg RUN_FROM="${EMQX_RUNNER}" \
+       --build-arg EMQX_NAME="$PROFILE" \
+       --tag "emqx/$PROFILE:${PKG_VSN}" \
+       -f "${DOCKERFILE}" .
+}
+
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
@@ -148,6 +161,10 @@ case "$ARTIFACT" in
         make -C "deploy/packages/${PKGERDIR}" clean
         EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
         ;;
+
+    docker)
+        make_docker
+        ;;
     *)
         log "Unknown artifact $ARTIFACT"
         exit 1
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 0206dc031..c79508ec1 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -2,9 +2,6 @@ ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
 ARG RUN_FROM=alpine:3.12
 FROM ${BUILD_FROM} AS builder
 
-ARG QEMU_ARCH=x86_64
-COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
-
 RUN apk add --no-cache \
     git \
     curl \
@@ -43,10 +40,9 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \
     org.label-schema.vcs-url="https://github.com/emqx/emqx" \
     maintainer="EMQ X Team <support@emqx.io>"
 
-ARG QEMU_ARCH=x86_64
 ARG EMQX_NAME=emqx
 
-COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/
+COPY deploy/docker/docker-entrypoint.sh /usr/bin/
 COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
 
 RUN ln -s /opt/emqx/bin/* /usr/local/bin/

From 1627ef78e71b3077e893fc3168768f9d47bac6c3 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 14:18:27 +0800
Subject: [PATCH 28/46] ci(build_packages): update docker build

Signed-off-by: zhanghongtong <rory-z@outlook.com>
---
 .github/workflows/build_packages.yaml     | 84 +++++++++++++----------
 .github/workflows/run_automate_tests.yaml | 41 ++++-------
 .github/workflows/run_fvt_tests.yaml      | 10 +--
 3 files changed, 68 insertions(+), 67 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index aa93c8ba5..a7f691735 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -338,17 +338,6 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
-          - [amd64, x86_64]
-          - [arm64v8, aarch64]
-          - [arm32v7, arm]
-          - [i386, i386]
-          - [s390x, s390x]
-        exclude:
-          - profile: emqx-ee
-            arch: [i386, i386]
-          - profile: emqx-ee
-            arch: [s390x, s390x]
 
     steps:
     - uses: actions/download-artifact@v2
@@ -357,22 +346,56 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
       with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        image: tonistiigi/binfmt:latest
+        platforms: all
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ github.repository_owner }}/${{ matrix.profile }}
+        flavor: |
+          latest=${{ !github.event.release.prerelease }}
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=match,pattern=[v|e](.*),group=1
+        labels:
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+    - uses: docker/login-action@v1
+      if: github.event_name == 'release'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/build-push-action@v2
+      with:
+        push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
+    - uses: aws-actions/configure-aws-credentials@v1
+      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Push image to aws ecr
+      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
+      run: |
+        version=${GITHUB_REF##*/}
+        docker pull emqx/emqx:${version#v}
+        docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+        docker push public.ecr.aws/emqx/emqx:${version#v}
 
   delete-artifact:
     runs-on: ubuntu-20.04
@@ -452,15 +475,6 @@ jobs:
              -X POST \
              -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
              ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release' && matrix.profile == 'emqx'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
     - name: push docker image to aws ecr
       if: github.event_name == 'release'
       run: |
diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml
index 06fd01c4b..c5e855a73 100644
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@@ -36,18 +36,20 @@ jobs:
           echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
           make deps-emqx-ee
           make clean
-        fi
-        make docker
-        echo "::set-output name=version::$(./pkg-vsn.sh)"
-        if [ -f EMQX_ENTERPRISE ]; then
+          make emqx-ee-docker
           echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar
         else
+          make emqx-docker
           echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar
         fi
     - uses: actions/upload-artifact@v2
       with:
-        name: emqx-docker-image-zip
-        path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip
+        name: emqx-docker-image
+        path: emqx.tar
 
   webhook:
     runs-on: ubuntu-latest
@@ -63,15 +65,10 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname}}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -166,15 +163,10 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -276,15 +268,10 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
     - name: docker compose up
       timeout-minutes: 5
       env:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 908d1fe33..9efbc0bb1 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -17,7 +17,7 @@ jobs:
           id: install_erlang
           with:
             otp-version: 23.2
-        - name: prepare
+        - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
@@ -26,12 +26,12 @@ jobs:
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -79,11 +79,11 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: install k3s
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"

From 6d0b6d2896405e3045cdef6ff480c0cacc780609 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 15:00:15 +0800
Subject: [PATCH 29/46] chore: delete needless docker.mk

---
 docker.mk | 188 ------------------------------------------------------
 1 file changed, 188 deletions(-)
 delete mode 100644 docker.mk

diff --git a/docker.mk b/docker.mk
deleted file mode 100644
index 7189ecc24..000000000
--- a/docker.mk
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-## default globals.
-## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise)
-## or the TARGET varialbe can be set beforehand to force a different name
-TARGET ?= emqx/$(PROFILE)
-QEMU_ARCH ?= x86_64
-ARCH ?= amd64
-QEMU_VERSION ?= v5.0.0-2
-OS ?= alpine
-export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-
-ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee)
-	ARCH_LIST := amd64 arm64v8 arm32v7
-	EMQX_NAME := emqx-ee
-else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge)
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx-edge
-else
-	ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x
-	EMQX_NAME := emqx
-endif
-
-.PHONY: docker
-docker: docker-build docker-tag docker-save
-
-.PHONY: docker-prepare
-docker-prepare:
-	## Prepare the machine before any code installation scripts
-	# @echo "PREPARE: Setting up dependencies."
-	# @apt update -y
-	# @apt install --only-upgrade docker-ce -y
-
-	## Update docker configuration to enable docker manifest command
-	@echo "PREPARE: Updating docker configuration"
-	@mkdir -p $$HOME/.docker
-
-	# enable experimental to use docker manifest command
-	@echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json
-	# enable experimental
-	@echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json
-	@service docker restart
-
-.PHONY: docker-build
-docker-build:
-	## Build Docker image
-	@echo "DOCKER BUILD: Build Docker image."
-	@echo "DOCKER BUILD: build version -> $(PKG_VSN)."
-	@echo "DOCKER BUILD: arch - $(ARCH)."
-	@echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)."
-	@echo "DOCKER BUILD: docker repo - $(TARGET) "
-	@echo "DOCKER BUILD: emqx name - $(EMQX_NAME)."
-
-	## Prepare qemu to build images other then x86_64 on travis
-	@echo "PREPARE: Qemu" \
-	&& docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-	@mkdir -p tmp \
-	&& cd tmp \
-	&& curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \
-	&& cd -
-
-	@docker build --no-cache \
-		--build-arg PKG_VSN=$(PKG_VSN)  \
-		--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine  \
-		--build-arg RUN_FROM=$(ARCH)/alpine:3.12 \
-		--build-arg EMQX_NAME=$(EMQX_NAME) \
-		--build-arg QEMU_ARCH=$(QEMU_ARCH) \
-		--tag $(TARGET):build-$(OS)-$(ARCH) \
-		-f deploy/docker/Dockerfile .
-
-.PHONY: docker-tag
-docker-tag:
-	@echo "DOCKER TAG: Tag Docker image."
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \
-			docker tag $(TARGET):build-$(OS)-$${arch}  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-			echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \
-			if [ $${arch} = amd64 ]; then \
-				docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \
-				echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \
-			fi; \
-		fi; \
-	done
-
-.PHONY: docker-save
-docker-save:
-	@echo "DOCKER SAVE: Save Docker image."
-
-	@mkdir -p _packages/$(EMQX_NAME)
-
-	@if [ -n  "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \
-		zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \
-		mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \
-	fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n  "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker save  $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \
-			mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \
-		fi; \
-	done
-
-.PHONY: docker-push
-docker-push:
-	@echo "DOCKER PUSH: Push Docker image.";
-	@echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN).";
-
-	@if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \
-		docker push $(TARGET):$(PKG_VSN); \
-		docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \
-		docker push $(TARGET):latest; \
-	fi;
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker push  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \
-		fi; \
-	done
-
-.PHONY: docker-manifest-list
-docker-manifest-list:
-	version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \
-	latest="docker manifest create --amend $(TARGET):latest"; \
-	for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \
-			version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-			latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \
-		fi; \
-	done; \
-	eval $$version; \
-	eval $$latest;
-
-	for arch in $(ARCH_LIST); do \
-		case $${arch} in \
-			"amd64") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \
-				fi; \
-				;; \
-			"arm64v8") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \
-				fi; \
-				;; \
-			"arm32v7") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \
-				fi; \
-				;; \
-			"i386") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \
-				fi; \
-				;; \
-			"s390x") \
-				if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-					docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-					docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \
-				fi; \
-				;; \
-		esac; \
-	done;
-
-	docker manifest inspect $(TARGET):$(PKG_VSN)
-	docker manifest push $(TARGET):$(PKG_VSN);
-	docker manifest inspect $(TARGET):latest
-	docker manifest push $(TARGET):latest;
-
-.PHONY: docker-clean
-docker-clean:
-	@echo "DOCKER CLEAN: Clean Docker image."
-
-	@if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)); fi
-
-	@for arch in $(ARCH_LIST); do \
-		if [ -n "$$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \
-			docker rmi -f $$(docker images -q  $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \
-		fi \
-	done

From 514634e541a57e070cad77ff2f1b99a1754f5b41 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 16:32:32 +0800
Subject: [PATCH 30/46] ci(build_packages): if it's enterprise, use
 Dockerfile.enterprise

---
 .github/workflows/build_packages.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index a7f691735..032011056 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -369,6 +369,7 @@ jobs:
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_TOKEN }}
     - uses: docker/build-push-action@v2
+      if: matrix.profile != 'emqx-ee'
       with:
         push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
         pull: true
@@ -382,6 +383,21 @@ jobs:
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
         context: source
+    - uses: docker/build-push-action@v2
+      if: matrix.profile == 'emqx-ee'
+      with:
+        push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile.enterprise
+        context: source
     - uses: aws-actions/configure-aws-credentials@v1
       if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
       with:

From 387af5295c3754ab2854e76e1b9a9749c3898a8b Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 17:56:58 +0800
Subject: [PATCH 31/46] chore(cts): ignore ldap error

---
 .github/workflows/run_cts_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml
index 84aa301c4..a069e143d 100644
--- a/.github/workflows/run_cts_tests.yaml
+++ b/.github/workflows/run_cts_tests.yaml
@@ -52,7 +52,7 @@ jobs:
           docker exec -i \
             -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
             --env-file .env \
-            erlang sh -c "make apps/emqx_auth_ldap-ct"
+            erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

From 6a5c37dab72f8d57243a8144696784511fe942d2 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 4 Jan 2022 21:02:40 +0800
Subject: [PATCH 32/46] ci(its): fix error

---
 .github/workflows/run_automate_tests.yaml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml
index c5e855a73..99a8dd55c 100644
--- a/.github/workflows/run_automate_tests.yaml
+++ b/.github/workflows/run_automate_tests.yaml
@@ -362,15 +362,10 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-docker-image-zip
+        name: emqx-docker-image
         path: /tmp
     - name: load docker image
-      env:
-        imgname: ${{ needs.build.outputs.imgname }}
-        version: ${{ needs.build.outputs.version }}
-      run: |
-        unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp
-        docker load < /tmp/${imgname}-docker-${version}
+      run: docker load < /tmp/emqx.tar
     - name: docker compose up
       timeout-minutes: 5
       env:

From 0b12a7e3a2c2a7bef851cdc9851f3fd1bf36ea72 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Wed, 5 Jan 2022 10:12:27 +0800
Subject: [PATCH 33/46] ci(build_packages): only push emqx broker to aws ecr

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 032011056..3ab1f7086 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -492,7 +492,7 @@ jobs:
              -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
              ${{ secrets.EMQX_IO_RELEASE_API }}
     - name: push docker image to aws ecr
-      if: github.event_name == 'release'
+      if: github.event_name == 'release' && matrix.profile == 'emqx'
       run: |
         set -e -x -u
         aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws

From ced2a4a11728ab027fc5a718845db43f6f52b339 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Wed, 5 Jan 2022 11:13:16 +0800
Subject: [PATCH 34/46] ci(static checks): do not run static check

---
 .github/workflows/run_test_cases.yaml | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index daabb0b52..9a40964bc 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -8,23 +8,6 @@ on:
   pull_request:
 
 jobs:
-    run_static_analysis:
-        runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
-
-        steps:
-        - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
-        - name: xref
-          run: make xref
-        - name: dialyzer
-          run: make dialyzer
-
     run_proper_test:
         runs-on: ubuntu-20.04
         container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04

From 9cf03bb18a1977f157169d4b386551e6c0bdec0b Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 5 Jan 2022 10:45:51 +0800
Subject: [PATCH 35/46] fix(mgmt): fix substring matching alg

---
 .../src/emqx_mgmt_api_clients.erl             | 40 +++++++++++--------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 1ddd87a3d..eaa6bf601 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -333,31 +333,24 @@ query({Qs, Fuzzy}, Start, Limit) ->
 
 match_fun(Ms, Fuzzy) ->
     MsC = ets:match_spec_compile(Ms),
-    REFuzzy = lists:map(fun({K, like, S}) ->
-                  {ok, RE} = re:compile(escape(S)),
-                  {K, like, RE}
-              end, Fuzzy),
     fun(Rows) ->
          case ets:match_spec_run(Rows, MsC) of
              [] -> [];
              Ls ->
                  lists:filter(fun(E) ->
-                    run_fuzzy_match(E, REFuzzy)
+                    run_fuzzy_match(E, Fuzzy)
                  end, Ls)
          end
     end.
 
-escape(B) when is_binary(B) ->
-    re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]).
-
 run_fuzzy_match(_, []) ->
     true;
-run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) ->
-    Val = case maps:get(Key, ClientInfo, "") of
-              undefined -> "";
+run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) ->
+    Val = case maps:get(Key, ClientInfo, undefined) of
+              undefined -> <<>>;
               V -> V
           end,
-    re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy).
+    binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy).
 
 %%--------------------------------------------------------------------
 %% QueryString to Match Spec
@@ -453,9 +446,24 @@ params2qs_test() ->
 
     [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]).
 
-escape_test() ->
-    Str = <<"\\n">>,
-    {ok, Re} = re:compile(escape(Str)),
-    {match, _} = re:run(<<"\\name">>, Re).
+fuzzy_match_test() ->
+    Info = {emqx_channel_info,
+            #{clientinfo =>
+              #{ clientid => <<"abcde">>
+               , username => <<"abc\\name*[]()">>
+               }}, []
+           },
+    true = run_fuzzy_match(Info, [{clientid, like, <<"abcde">>}]),
+    true = run_fuzzy_match(Info, [{clientid, like, <<"bcd">>}]),
+    false = run_fuzzy_match(Info, [{clientid, like, <<"defh">>}]),
+
+    true = run_fuzzy_match(Info, [{username, like, <<"\\name">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"*">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"[]">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"()">>}]),
+    false = run_fuzzy_match(Info, [{username, like, <<"))">>}]),
+
+    true = run_fuzzy_match(Info, [{clientid, like, <<"de">>},
+                                  {username, like, <<"[]">>}]).
 
 -endif.

From d6f56cbcbe3f783eca89ef45dbaa20da839095af Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 5 Jan 2022 11:28:11 +0800
Subject: [PATCH 36/46] fix(mgmt): convert timestamp to milliseconds

---
 apps/emqx_management/src/emqx_mgmt_api.erl         | 8 +++++---
 apps/emqx_management/src/emqx_mgmt_api_clients.erl | 8 ++++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index e068c5384..c34791b57 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -297,10 +297,12 @@ to_integer(I) when is_integer(I) ->
 to_integer(B) when is_binary(B) ->
     binary_to_integer(B).
 
+%% @doc The input timestamp time is in seconds, which needs to be
+%% converted to internal milliseconds here
 to_timestamp(I) when is_integer(I) ->
-    I;
+    I * 1000;
 to_timestamp(B) when is_binary(B) ->
-    binary_to_integer(B).
+    binary_to_integer(B) * 1000.
 
 aton(B) when is_binary(B) ->
     list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]).
@@ -332,7 +334,7 @@ params2qs_test() ->
     ExpectedQs = [{str, '=:=', <<"abc">>},
                   {int, '=:=', 123},
                   {atom, '=:=', connected},
-                  {ts, '=:=', 156000},
+                  {ts, '=:=', 156000000},
                   {range, '>=', 1, '=<', 5}
                  ],
     FuzzyQs = [{fuzzy, like, <<"user">>},
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index eaa6bf601..7d3dbddc8 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -435,10 +435,10 @@ params2qs_test() ->
                         proto_ver => 4,
                         connected_at => '$3'},
           session => #{created_at => '$2'}},
-    ExpectedCondi = [{'>=','$2', 1},
-                     {'=<','$2', 5},
-                     {'>=','$3', 1},
-                     {'=<','$3', 5}],
+    ExpectedCondi = [{'>=','$2', 1000},
+                     {'=<','$2', 5000},
+                     {'>=','$3', 1000},
+                     {'=<','$3', 5000}],
     {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema),
     [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1),
     ?assertEqual(ExpectedMtchHead, MtchHead),

From b7bcb37eabb6e6de427d0254d0fae9544308fefb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 6 Jan 2022 20:18:05 +0800
Subject: [PATCH 37/46] fix(rule): restore metrics for actions

---
 .../src/emqx_rule_engine.app.src              |   2 +-
 .../src/emqx_rule_engine.appup.src            | 121 +++++++++++-------
 .../emqx_rule_engine/src/emqx_rule_engine.erl |  13 ++
 3 files changed, 88 insertions(+), 48 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 66af0da21..dd39cfc40 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.6"}, % strict semver, bump manually!
+  {vsn, "4.3.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index c94dc994d..2c2a5c4cb 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,52 +1,79 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.5",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
-    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.3",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [
+   {"4.3.6",
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.5",
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
    {"4.3.4",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}],
-  [{"4.3.5",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
-    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    ]},
    {"4.3.3",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.2",
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.1",
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.0",
+    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {<<".*">>, []}
+  ],
+  [
+   {"4.3.6",
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.5",
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
    {"4.3.4",
-    [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}]}.
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.3",
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.2",
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.1",
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.0",
+    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
+      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    ]},
+   {<<".*">>, []}
+  ]
+}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
index 6c8b95064..0dabb1c52 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
@@ -491,11 +491,24 @@ may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) ->
 may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) ->
     %% prepare new actions before removing old ones
     NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)),
+    ok = restore_action_metrics(OldActions, NewActions),
     _ = ?CLUSTER_CALL(clear_actions, [OldActions]),
     may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params));
 may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params
     Rule.
 
+%% NOTE: if the user removed an action, but the action is not the last one in the list,
+%% the `restore_action_metrics/2` will not work as expected!
+restore_action_metrics([#action_instance{id = OldId} | OldActions],
+                       [#action_instance{id = NewId} | NewActions]) ->
+    emqx_rule_metrics:inc_actions_taken(NewId, emqx_rule_metrics:get_actions_taken(OldId)),
+    emqx_rule_metrics:inc_actions_success(NewId, emqx_rule_metrics:get_actions_success(OldId)),
+    emqx_rule_metrics:inc_actions_error(NewId, emqx_rule_metrics:get_actions_error(OldId)),
+    emqx_rule_metrics:inc_actions_exception(NewId, emqx_rule_metrics:get_actions_exception(OldId)),
+    restore_action_metrics(OldActions, NewActions);
+restore_action_metrics(_, _) ->
+    ok.
+
 ignore_lib_apps(Apps) ->
     LibApps = [kernel, stdlib, sasl, appmon, eldap, erts,
                syntax_tools, ssl, crypto, mnesia, os_mon,

From fbeba39f0d0cfd6f5222804b4a0d2bc498ea62a8 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 6 Jan 2022 20:29:35 +0800
Subject: [PATCH 38/46] fix(rule): compare to null variables should return
 false

---
 .../src/emqx_rule_engine.appup.src            | 86 +++++++++++--------
 .../src/emqx_rule_runtime.erl                 |  4 +
 2 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 2c2a5c4cb..d97c6542f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -3,76 +3,90 @@
   [
    {"4.3.6",
     [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.5",
     [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.4",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.3",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.2",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.1",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.0",
-    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {<<".*">>, []}
   ],
   [
    {"4.3.6",
     [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.5",
     [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.4",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.3",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.2",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.1",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {"4.3.0",
-    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
     ]},
    {<<".*">>, []}
   ]
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index f9e210ab3..080498d82 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -204,6 +204,10 @@ match_conditions({}, _Data) ->
     true.
 
 %% comparing numbers against strings
+compare(Op, undefined, undefined) ->
+    do_compare(Op, undefined, undefined);
+compare(_Op, L, R) when L == undefined; R == undefined ->
+    false;
 compare(Op, L, R) when is_number(L), is_binary(R) ->
     do_compare(Op, L, number(R));
 compare(Op, L, R) when is_binary(L), is_number(R) ->

From 5c29c20426baea10ffb84915788b3bbb4e674012 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 20 Dec 2021 16:44:51 +0800
Subject: [PATCH 39/46] fix(telemetry): use required fields, rolling distro use
 PRETTY_NAME

---
 lib-ce/emqx_telemetry/src/emqx_telemetry.erl | 38 +++++++++-----------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl
index 3e8d3ffd3..e07eeb7fb 100644
--- a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl
+++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl
@@ -82,7 +82,7 @@
           timer = undefined :: undefined | reference()
         }).
 
-%% The count of 100-nanosecond intervals between the UUID epoch 
+%% The count of 100-nanosecond intervals between the UUID epoch
 %% 1582-10-15 00:00:00 and the UNIX epoch 1970-01-01 00:00:00.
 -define(GREGORIAN_EPOCH_OFFSET, 16#01b21dd213814000).
 
@@ -253,29 +253,23 @@ os_info() ->
             [{os_name, Name},
              {os_version, Version}];
         {unix, _} ->
-            case file:read_file_info("/etc/os-release") of
+            case file:read_file("/etc/os-release") of
                 {error, _} ->
                     [{os_name, "Unknown"},
                      {os_version, "Unknown"}];
-                {ok, FileInfo} ->
-                    case FileInfo#file_info.access of
-                        Access when Access =:= read orelse Access =:= read_write ->
-                            OSInfo = lists:foldl(fun(Line, Acc) ->
-                                                     [Var, Value] = string:tokens(Line, "="),
-                                                     NValue = case Value of
-                                                                  _ when is_list(Value) ->
-                                                                      lists:nth(1, string:tokens(Value, "\""));
-                                                                  _ ->
-                                                                      Value
-                                                              end,
-                                                     [{Var, NValue} | Acc]
-                                                 end, [], string:tokens(os:cmd("cat /etc/os-release"), "\n")),
-                            [{os_name, get_value("NAME", OSInfo)},
-                             {os_version, get_value("VERSION", OSInfo, get_value("VERSION_ID", OSInfo))}];
-                        _ ->
-                            [{os_name, "Unknown"},
-                             {os_version, "Unknown"}]
-                    end
+                {ok, FileContent} ->
+                    OSInfo = lists:foldl(fun(Line, Acc) ->
+                                                 [Var, Value] = string:tokens(Line, "="),
+                                                 NValue = case Value of
+                                                              _ when is_list(Value) ->
+                                                                  lists:nth(1, string:tokens(Value, "\""));
+                                                              _ ->
+                                                                  Value
+                                                          end,
+                                                 [{Var, NValue} | Acc]
+                                         end, [], string:tokens(binary:bin_to_list(FileContent), "\n")),
+                    [{os_name, get_value("NAME", OSInfo)},
+                     {os_version, get_value("VERSION", OSInfo, get_value("VERSION_ID", OSInfo, get_value("PRETTY_NAME", OSInfo)))}]
             end;
         {win32, nt} ->
             Ver = os:cmd("ver"),
@@ -429,5 +423,7 @@ module_attributes(Module) ->
 
 bin(L) when is_list(L) ->
     list_to_binary(L);
+bin(A) when is_atom(A) ->
+    atom_to_binary(A);
 bin(B) when is_binary(B) ->
     B.

From 2fba756aea96ff815997f0bbc8820d946a5c2a64 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 20 Dec 2021 18:31:20 +0800
Subject: [PATCH 40/46] chore(telemetry): update appup.src

---
 lib-ce/emqx_telemetry/src/emqx_telemetry.app.src   | 2 +-
 lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src
index e65678c37..8394836ef 100644
--- a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src
+++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src
@@ -1,6 +1,6 @@
 {application, emqx_telemetry,
  [{description, "EMQ X Telemetry"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_telemetry_sup]},
   {applications, [kernel,stdlib]},
diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src
index 27998f0d5..23de107d9 100644
--- a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src
+++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src
@@ -1,13 +1,13 @@
 %% -*- mode: erlang -*-
 {VSN,
  [
-   {"4.3.0", [
+   {<<"4\\.3\\.[0-1]">>, [
      {load_module, emqx_telemetry, brutal_purge, soft_purge, []}
    ]},
    {<<".*">>, []}
  ],
  [
-   {"4.3.0", [
+   {<<"4\\.3\\.[0-1]">>, [
      {load_module, emqx_telemetry, brutal_purge, soft_purge, []}
    ]},
    {<<".*">>, []}

From 5424588a72c4c7659b525ee988814b2c18878551 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 11 Jan 2022 11:11:30 +0800
Subject: [PATCH 41/46] chore: remove unwanted appup commands that came with
 merge

---
 .../src/emqx_management.appup.src             | 16 +---
 .../emqx_retainer/src/emqx_retainer.appup.src | 15 +--
 .../src/emqx_rule_engine.appup.src            | 92 +------------------
 3 files changed, 7 insertions(+), 116 deletions(-)

diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src
index 1463334b4..109eeef80 100644
--- a/apps/emqx_management/src/emqx_management.appup.src
+++ b/apps/emqx_management/src/emqx_management.appup.src
@@ -1,17 +1,5 @@
 %% -*- mode: erlang -*-
 {VSN,
- [ {<<"4\\.3\\.[0-9]+">>,
-    [ {apply,{minirest,stop_http,['http:management']}},
-      {apply,{minirest,stop_http,['https:management']}},
-      {restart_application, emqx_management}
-    ]},
-   {<<".*">>, []}
- ],
- [ {<<"4\\.3\\.[0-9]+">>,
-    [ {apply,{minirest,stop_http,['http:management']}},
-      {apply,{minirest,stop_http,['https:management']}},
-      {restart_application, emqx_management}
-    ]},
-   {<<".*">>, []}
- ]
+  [{<<".*">>,[]}],
+  [{<<".*">>,[]}]
 }.
diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src
index 45ec6420c..109eeef80 100644
--- a/apps/emqx_retainer/src/emqx_retainer.appup.src
+++ b/apps/emqx_retainer/src/emqx_retainer.appup.src
@@ -1,14 +1,5 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.2",
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[0-1]">>,
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
-     {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}],
-  [{"4.3.2",
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[0-1]">>,
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
-     {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}]}.
+  [{<<".*">>,[]}],
+  [{<<".*">>,[]}]
+}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index d97c6542f..109eeef80 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,93 +1,5 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [
-   {"4.3.6",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.5",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.4",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.3",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.2",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.1",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.0",
-    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {<<".*">>, []}
-  ],
-  [
-   {"4.3.6",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.5",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.4",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.3",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.2",
-    [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.1",
-    [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.0",
-    [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
-    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
-    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-    ]},
-   {<<".*">>, []}
-  ]
+  [{<<".*">>,[]}],
+  [{<<".*">>,[]}]
 }.

From 9aa178b51d074788f5079c0b0c0775caadf39396 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 11 Jan 2022 14:46:20 +0800
Subject: [PATCH 42/46] fix(stomp): fix parsing rear frame split byte crash

fix: https://github.com/emqx/emqx/issues/6693
---
 apps/emqx_stomp/src/emqx_stomp_frame.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_stomp/src/emqx_stomp_frame.erl b/apps/emqx_stomp/src/emqx_stomp_frame.erl
index c4d19ae3a..212242969 100644
--- a/apps/emqx_stomp/src/emqx_stomp_frame.erl
+++ b/apps/emqx_stomp/src/emqx_stomp_frame.erl
@@ -121,7 +121,7 @@ g(Key, Opts, Val) ->
 parse(<<>>, Parser) ->
     {more, Parser};
 
-parse(Bytes, #{phase := body, len := Len, state := State}) ->
+parse(Bytes, #{phase := body, length := Len, state := State}) ->
     parse(body, Bytes, State, Len);
 
 parse(Bytes, Parser = #{pre := Pre}) ->

From 9efab7d5df0e620cf57cb91be45898c9cc5718ab Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 11 Jan 2022 14:48:07 +0800
Subject: [PATCH 43/46] chore(stomp): update appup & app.src

---
 apps/emqx_stomp/src/emqx_stomp.app.src   |  2 +-
 apps/emqx_stomp/src/emqx_stomp.appup.src | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src
index c2f4b57d3..a70826e34 100644
--- a/apps/emqx_stomp/src/emqx_stomp.app.src
+++ b/apps/emqx_stomp/src/emqx_stomp.app.src
@@ -1,6 +1,6 @@
 {application, emqx_stomp,
  [{description, "EMQ X Stomp Protocol Plugin"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_stomp_sup]},
   {applications, [kernel,stdlib]},
diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src
index 0b5372dc9..8d61073d0 100644
--- a/apps/emqx_stomp/src/emqx_stomp.appup.src
+++ b/apps/emqx_stomp/src/emqx_stomp.appup.src
@@ -1,16 +1,24 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]},
+  [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{restart_application,emqx_stomp},
      {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]},
    {<<".*">>,[]}],
-  [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]},
+  [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{restart_application,emqx_stomp}]},

From f2cfde7421a37e54295f44ced554ea03fc8f8164 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 11 Jan 2022 15:54:37 +0800
Subject: [PATCH 44/46] fix(mgmt): fix subscription info format

---
 .gitignore                                    |  1 +
 apps/emqx_management/src/emqx_mgmt.erl        | 24 ++++++++++---------
 .../src/emqx_mgmt_api_subscriptions.erl       |  4 ++--
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.gitignore b/.gitignore
index bbb4edebb..13217dc00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,3 +59,4 @@ erlang_ls.config
 *#
 # For direnv
 .envrc
+mix.lock
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 39595b104..e68a6163f 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -58,8 +58,8 @@
 -export([ list_subscriptions/1
         , list_subscriptions_via_topic/2
         , list_subscriptions_via_topic/3
-        , lookup_subscriptions/1
         , lookup_subscriptions/2
+        , lookup_subscriptions/3
         ]).
 
 %% Routes
@@ -322,18 +322,20 @@ list_subscriptions_via_topic(Node, Topic, {M,F}) when Node =:= node() ->
 list_subscriptions_via_topic(Node, Topic, FormatFun) ->
     rpc_call(Node, list_subscriptions_via_topic, [Node, Topic, FormatFun]).
 
-lookup_subscriptions(ClientId) ->
-    lists:append([lookup_subscriptions(Node, ClientId) || Node <- ekka_mnesia:running_nodes()]).
+lookup_subscriptions(ClientId, FormatFun) ->
+    lists:append([lookup_subscriptions(Node, ClientId, FormatFun) || Node <- ekka_mnesia:running_nodes()]).
 
-lookup_subscriptions(Node, ClientId) when Node =:= node() ->
-    case ets:lookup(emqx_subid, ClientId) of
-        [] -> [];
-        [{_, Pid}] ->
-            ets:match_object(emqx_suboption, {{Pid, '_'}, '_'})
-    end;
+lookup_subscriptions(Node, ClientId, {M, F}) when Node =:= node() ->
+    Result = case ets:lookup(emqx_subid, ClientId) of
+                 [] -> [];
+                 [{_, Pid}] ->
+                     ets:match_object(emqx_suboption, {{Pid, '_'}, '_'})
+             end,
+    %% format at the called node
+    erlang:apply(M, F, [Result]);
 
-lookup_subscriptions(Node, ClientId) ->
-    rpc_call(Node, lookup_subscriptions, [Node, ClientId]).
+lookup_subscriptions(Node, ClientId, FormatFun) ->
+    rpc_call(Node, lookup_subscriptions, [Node, ClientId, FormatFun]).
 
 %%--------------------------------------------------------------------
 %% Routes
diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index 4165ca51a..464df33c9 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -85,10 +85,10 @@ list(#{node := Node} = Bindings, Params) ->
     end.
 
 lookup(#{node := Node, clientid := ClientId}, _Params) ->
-    minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))});
+    minirest:return({ok, emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId), ?format_fun)});
 
 lookup(#{clientid := ClientId}, _Params) ->
-    minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}).
+    minirest:return({ok, emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId), ?format_fun)}).
 
 format(Items) when is_list(Items) ->
     [format(Item) || Item <- Items];

From 47ba72729fc6400fc0dc2c4697f14c3a90c4540e Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 11 Jan 2022 16:23:25 +0800
Subject: [PATCH 45/46] chore(appup): update appup.src

---
 apps/emqx_management/src/emqx_management.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index e8b235be7..bee65781a 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.10"}, % strict semver, bump manually!
+  {vsn, "4.3.11"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},

From 556a81a80e180fbfb5be5bfa926c0f30d8fa44b1 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 12 Jan 2022 10:09:51 +0800
Subject: [PATCH 46/46] chore(appup): app version updated after last tag

---
 apps/emqx_management/src/emqx_management.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index bee65781a..e8b235be7 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.11"}, % strict semver, bump manually!
+  {vsn, "4.3.10"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},