From 0b52097c789f45182db4b0a945cf591a2593ef43 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 27 May 2022 16:51:05 +0800
Subject: [PATCH 01/25] feat(rewrite): Support %u and %c placeholders in topic
 rewrite rules

fix #7438
---
 lib-ce/emqx_modules/src/emqx_mod_rewrite.erl  | 41 +++++++++++++------
 .../test/emqx_mod_rewrite_SUITE.erl           | 31 ++++++++------
 2 files changed, 48 insertions(+), 24 deletions(-)

diff --git a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl
index b7068f4df..b51d29291 100644
--- a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl
+++ b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl
@@ -23,7 +23,7 @@
 
 -ifdef(TEST).
 -export([ compile/1
-        , match_and_rewrite/2
+        , match_and_rewrite/3
         ]).
 -endif.
 
@@ -49,14 +49,17 @@ load(RawRules) ->
     emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}),
     emqx_hooks:put('message.publish',    {?MODULE, rewrite_publish, [PubRules]}).
 
-rewrite_subscribe(_ClientInfo, _Properties, TopicFilters, Rules) ->
-    {ok, [{match_and_rewrite(Topic, Rules), Opts} || {Topic, Opts} <- TopicFilters]}.
+rewrite_subscribe(ClientInfo, _Properties, TopicFilters, Rules) ->
+    Binds = fill_client_binds(ClientInfo),
+    {ok, [{match_and_rewrite(Topic, Rules, Binds), Opts} || {Topic, Opts} <- TopicFilters]}.
 
-rewrite_unsubscribe(_ClientInfo, _Properties, TopicFilters, Rules) ->
-    {ok, [{match_and_rewrite(Topic, Rules), Opts} || {Topic, Opts} <- TopicFilters]}.
+rewrite_unsubscribe(ClientInfo, _Properties, TopicFilters, Rules) ->
+    Binds = fill_client_binds(ClientInfo),
+    {ok, [{match_and_rewrite(Topic, Rules, Binds), Opts} || {Topic, Opts} <- TopicFilters]}.
 
 rewrite_publish(Message = #message{topic = Topic}, Rules) ->
-    {ok, Message#message{topic = match_and_rewrite(Topic, Rules)}}.
+    Binds = fill_client_binds(Message),
+    {ok, Message#message{topic = match_and_rewrite(Topic, Rules, Binds)}}.
 
 unload(_) ->
     emqx_hooks:del('client.subscribe',   {?MODULE, rewrite_subscribe}),
@@ -80,16 +83,16 @@ compile(Rules) ->
                  end || {rewrite, sub, Topic, Re, Dest}<- Rules ],
     {PubRules, SubRules}.
 
-match_and_rewrite(Topic, []) ->
+match_and_rewrite(Topic, [], _) ->
     Topic;
 
-match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules]) ->
+match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules], Binds) ->
     case emqx_topic:match(Topic, Filter) of
-        true  -> rewrite(Topic, MP, Dest);
-        false -> match_and_rewrite(Topic, Rules)
+        true  -> rewrite(Topic, MP, Dest, Binds);
+        false -> match_and_rewrite(Topic, Rules, Binds)
     end.
 
-rewrite(Topic, MP, Dest) ->
+rewrite(Topic, MP, Dest, Binds) ->
     case re:run(Topic, MP, [{capture, all_but_first, list}]) of
         {match, Captured} ->
             Vars = lists:zip(["\\$" ++ integer_to_list(I)
@@ -97,7 +100,21 @@ rewrite(Topic, MP, Dest) ->
             iolist_to_binary(lists:foldl(
                     fun({Var, Val}, Acc) ->
                         re:replace(Acc, Var, Val, [global])
-                    end, Dest, Vars));
+                    end, Dest, Binds ++ Vars));
         nomatch -> Topic
     end.
 
+fill_client_binds(#{clientid := ClientId, username := Username}) ->
+    filter_client_binds([{"%c", ClientId}, {"%u", Username}]);
+
+fill_client_binds(#message{from = ClientId, headers = Headers}) ->
+    Username = maps:get(username, Headers, undefined),
+    filter_client_binds([{"%c", ClientId}, {"%u", Username}]).
+
+filter_client_binds(Binds) ->
+    lists:filter(fun({_, undefined}) -> false;
+                    ({_, <<"">>}) -> false;
+                    ({_, ""}) -> false;
+                    (_) -> true
+                   end,
+                 Binds).
diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl
index 466f4a3f8..44574344b 100644
--- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl
+++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl
@@ -23,7 +23,11 @@
 -include_lib("eunit/include/eunit.hrl").
 
 -define(RULES, [{rewrite, pub, <<"x/#">>,<<"^x/y/(.+)$">>,<<"z/y/$1">>},
-                {rewrite, sub, <<"y/+/z/#">>,<<"^y/(.+)/z/(.+)$">>,<<"y/z/$2">>}
+                {rewrite, pub, <<"name/#">>,<<"^name/(.+)$">>,<<"pub/%u/$1">>},
+                {rewrite, pub, <<"c/#">>,<<"^c/(.+)$">>,<<"pub/%c/$1">>},
+                {rewrite, sub, <<"y/+/z/#">>,<<"^y/(.+)/z/(.+)$">>,<<"y/z/$2">>},
+                {rewrite, sub, <<"name/#">>,<<"^name/(.+)$">>,<<"sub/%u/$1">>},
+                {rewrite, sub, <<"c/#">>,<<"^c/(.+)$">>,<<"sub/%c/$1">>}
                ]).
 
 all() -> emqx_ct:all(?MODULE).
@@ -41,16 +45,18 @@ end_per_suite(_Config) ->
 %% Test case for emqx_mod_write
 t_mod_rewrite(_Config) ->
     ok = emqx_mod_rewrite:load(?RULES),
-    {ok, C} = emqtt:start_link([{clientid, <<"rewrite_client">>}]),
+    {ok, C} = emqtt:start_link([{clientid, <<"c1">>}, {username , <<"u1">>}]),
     {ok, _} = emqtt:connect(C),
-    PubOrigTopics = [<<"x/y/2">>, <<"x/1/2">>],
-    PubDestTopics = [<<"z/y/2">>, <<"x/1/2">>],
-    SubOrigTopics = [<<"y/a/z/b">>, <<"y/def">>],
-    SubDestTopics = [<<"y/z/b">>, <<"y/def">>],
+
+    PubOrigTopics = [<<"x/y/2">>, <<"x/1/2">>, <<"name/1">>, <<"c/1">>],
+    PubDestTopics = [<<"z/y/2">>, <<"x/1/2">>, <<"pub/u1/1">>, <<"pub/c1/1">>],
+    SubOrigTopics = [<<"y/a/z/b">>, <<"y/def">>, <<"name/1">>, <<"c/1">>],
+    SubDestTopics = [<<"y/z/b">>, <<"y/def">>, <<"sub/u1/1">>, <<"sub/c1/1">>],
+
     %% Sub Rules
     {ok, _Props, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- SubOrigTopics]),
     timer:sleep(100),
-    Subscriptions = emqx_broker:subscriptions(<<"rewrite_client">>),
+    Subscriptions = emqx_broker:subscriptions(<<"c1">>),
     ?assertEqual(SubDestTopics, [Topic || {Topic, _SubOpts} <- Subscriptions]),
     RecvTopics1 = [begin
                       ok = emqtt:publish(C, Topic, <<"payload">>),
@@ -60,7 +66,8 @@ t_mod_rewrite(_Config) ->
     ?assertEqual(SubDestTopics, RecvTopics1),
     {ok, _, _} = emqtt:unsubscribe(C, SubOrigTopics),
     timer:sleep(100),
-    ?assertEqual([], emqx_broker:subscriptions(<<"rewrite_client">>)),
+    ?assertEqual([], emqx_broker:subscriptions(<<"c1">>)),
+
     %% Pub Rules
     {ok, _, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]),
     RecvTopics2 = [begin
@@ -76,10 +83,10 @@ t_mod_rewrite(_Config) ->
 
 t_rewrite_rule(_Config) ->
     {PubRules, SubRules} = emqx_mod_rewrite:compile(?RULES),
-    ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules)),
-    ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules)),
-    ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules)),
-    ?assertEqual(<<"y/def">>, emqx_mod_rewrite:match_and_rewrite(<<"y/def">>, SubRules)).
+    ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules, [])),
+    ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules, [])),
+    ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules, [])),
+    ?assertEqual(<<"y/def">>, emqx_mod_rewrite:match_and_rewrite(<<"y/def">>, SubRules, [])).
 
 %%--------------------------------------------------------------------
 %% Internal functions

From 1b658171a2a95c22671e303853dff856ef63f146 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 30 May 2022 10:36:31 +0800
Subject: [PATCH 02/25] chore(modules): bump version and update appup file

---
 lib-ce/emqx_modules/src/emqx_modules.app.src  |  2 +-
 .../emqx_modules/src/emqx_modules.appup.src   | 34 +++++++++++++------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src
index 361b11157..49af5d3ea 100644
--- a/lib-ce/emqx_modules/src/emqx_modules.app.src
+++ b/lib-ce/emqx_modules/src/emqx_modules.app.src
@@ -1,6 +1,6 @@
 {application, emqx_modules,
  [{description, "EMQ X Module Management"},
-  {vsn, "4.3.6"},
+  {vsn, "4.3.7"},
   {modules, []},
   {applications, [kernel,stdlib]},
   {mod, {emqx_modules_app, []}},
diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src
index fdbacbfc6..d32cc7286 100644
--- a/lib-ce/emqx_modules/src/emqx_modules.appup.src
+++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src
@@ -1,49 +1,63 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.5",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+  [{"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-3]">>,
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {update,emqx_mod_delayed,{advanced,[]}},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.5",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
+  [{"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-3]">>,
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {update,emqx_mod_delayed,{advanced,[]}},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.

From 50ff2edf131b814b9935bde62dcb38a93401cb7e Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 3 Jun 2022 11:18:52 -0300
Subject: [PATCH 03/25] feat(tls): add configs for private key password in
 dashboard/mgmt

---
 CHANGES-4.3.md                                   | 8 ++++++++
 apps/emqx_management/etc/emqx_management.conf    | 1 +
 apps/emqx_management/priv/emqx_management.schema | 5 +++++
 lib-ce/emqx_dashboard/etc/emqx_dashboard.conf    | 6 ++++++
 lib-ce/emqx_dashboard/priv/emqx_dashboard.schema | 5 +++++
 5 files changed, 25 insertions(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 2819f729a..00cb8cf1e 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -10,6 +10,14 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.16
+
+### Enhancements
+
+- Add the possibility of configuring the password for
+  password-protected private key files used for dashboard and
+  management HTTPS listeners. [#8129]
+
 ## v4.3.15
 
 ### Enhancements
diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf
index 0170059d7..f9e6a518c 100644
--- a/apps/emqx_management/etc/emqx_management.conf
+++ b/apps/emqx_management/etc/emqx_management.conf
@@ -43,6 +43,7 @@ management.listener.http.ipv6_v6only = false
 ## management.listener.https.send_timeout_close = on
 ## management.listener.https.certfile = etc/certs/cert.pem
 ## management.listener.https.keyfile = etc/certs/key.pem
+## management.listener.https.key_password = yourpass
 ## management.listener.https.cacertfile = etc/certs/cacert.pem
 ## management.listener.https.verify = verify_peer
 ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier
diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema
index a30a20e4d..e0cc47d2f 100644
--- a/apps/emqx_management/priv/emqx_management.schema
+++ b/apps/emqx_management/priv/emqx_management.schema
@@ -143,6 +143,10 @@
   {datatype, string}
 ]}.
 
+{mapping, "management.listener.https.key_password", "emqx_management.listeners", [
+  {datatype, string}
+]}.
+
 {mapping, "management.listener.https.certfile", "emqx_management.listeners", [
   {datatype, string}
 ]}.
@@ -217,6 +221,7 @@ end}.
                 Filter([{versions, Versions},
                         {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))},
                         {keyfile,    cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},
+                        {password,   cuttlefish:conf_get(Prefix ++ ".key_password", Conf, undefined)},
                         {certfile,   cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
                         {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)},
                         {verify,     cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)},
diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf
index 2d59264a1..f59f27a47 100644
--- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf
+++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf
@@ -82,6 +82,12 @@ dashboard.listener.http.ipv6_v6only = false
 ## Value: File
 ## dashboard.listener.https.keyfile = etc/certs/key.pem
 
+## String containing the private key file password. Only used if the
+## private keyfile is password-protected.
+##
+## Value: String
+## dashboard.listener.https.key_password = yourpass
+
 ## Path to a file containing the user certificate.
 ##
 ## Value: File
diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema
index a2985429b..43093c3ba 100644
--- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema
+++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema
@@ -74,6 +74,10 @@
   {datatype, string}
 ]}.
 
+{mapping, "dashboard.listener.https.key_password", "emqx_dashboard.listeners", [
+  {datatype, string}
+]}.
+
 {mapping, "dashboard.listener.https.certfile", "emqx_dashboard.listeners", [
   {datatype, string}
 ]}.
@@ -127,6 +131,7 @@
                         {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))},
                         {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)},
                         {keyfile,    cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},
+                        {password,   cuttlefish:conf_get(Prefix ++ ".key_password", Conf, undefined)},
                         {certfile,   cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
                         {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)},
                         {verify,     cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)},

From c52e2f1b51c4be6990e05c72916b6360130d8700 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 2 Jun 2022 13:45:00 +0200
Subject: [PATCH 04/25] ci: use erlef/setup-beam@v1 to install erlang

---
 .github/workflows/run_fvt_tests.yaml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 1eb189720..ef76e8108 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -13,10 +13,9 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
-          id: install_erlang
+        - uses: erlef/setup-beam@v1
           with:
-            otp-version: 23.2
+            otp-version: "23.2"
         - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -66,10 +65,9 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
-          id: install_erlang
+        - uses: erlef/setup-beam@v1
           with:
-            otp-version: 23.2
+            otp-version: "23.2"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then

From 681d3652c6af16ca709fa59be417369a197f4ac8 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 6 Jun 2022 16:16:24 +0200
Subject: [PATCH 05/25] chore: bump emqx_dashboard app vsn

---
 lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
index ab52c2928..d0577fa74 100644
--- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
@@ -1,6 +1,6 @@
 {application, emqx_dashboard,
  [{description, "EMQ X Web Dashboard"},
-  {vsn, "4.3.11"}, % strict semver, bump manually!
+  {vsn, "4.3.12"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_dashboard_sup]},
   {applications, [kernel,stdlib,mnesia,minirest]},

From 1acbe4f72110cbb9817fc7181be057e5d51923c3 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 6 Jun 2022 23:19:39 +0200
Subject: [PATCH 06/25] chore(emqx_rule_engine): bump app vsn and update appup

---
 apps/emqx_rule_engine/src/emqx_rule_engine.app.src   | 2 +-
 apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 62ead0c36..0d859a230 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.10"}, % strict semver, bump manually!
+  {vsn, "4.3.11"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 4c2f653d9..7b4049254 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.9",
+  [{"4.3.10",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.9",
     [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -135,7 +136,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.9",
+  [{"4.3.10",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.9",
     [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},

From 6ad1a92db0e002cbe06417de306c328dde7d3a93 Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Tue, 7 Jun 2022 14:45:03 +0800
Subject: [PATCH 07/25] fix: fix helm install failed when enable ingress

---
 deploy/charts/emqx/templates/ingress.dashboard.yaml | 2 +-
 deploy/charts/emqx/templates/ingress.mgmt.yaml      | 2 +-
 deploy/charts/emqx/templates/ingress.wss.yaml       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/deploy/charts/emqx/templates/ingress.dashboard.yaml b/deploy/charts/emqx/templates/ingress.dashboard.yaml
index 5a8bbd4a9..6b305334d 100644
--- a/deploy/charts/emqx/templates/ingress.dashboard.yaml
+++ b/deploy/charts/emqx/templates/ingress.dashboard.yaml
@@ -34,7 +34,7 @@ spec:
       paths:
       - path: /
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: {{ .Values.ingress.dashboard.pathType | default "ImplementationSpecific" }}
+        pathType: {{ $.Values.ingress.dashboard.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
diff --git a/deploy/charts/emqx/templates/ingress.mgmt.yaml b/deploy/charts/emqx/templates/ingress.mgmt.yaml
index 6c3811f77..bd0940098 100644
--- a/deploy/charts/emqx/templates/ingress.mgmt.yaml
+++ b/deploy/charts/emqx/templates/ingress.mgmt.yaml
@@ -34,7 +34,7 @@ spec:
       paths:
       - path: {{ $.Values.ingress.mgmt.path | default "/" }}
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: {{ .Values.ingress.mgmt.pathType | default "ImplementationSpecific" }}
+        pathType: {{ $.Values.ingress.mgmt.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
diff --git a/deploy/charts/emqx/templates/ingress.wss.yaml b/deploy/charts/emqx/templates/ingress.wss.yaml
index ec74889bc..86c12abc1 100644
--- a/deploy/charts/emqx/templates/ingress.wss.yaml
+++ b/deploy/charts/emqx/templates/ingress.wss.yaml
@@ -34,7 +34,7 @@ spec:
       paths:
       - path: {{ $.Values.ingress.wss.path | default "/mqtt" }}
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: {{ .Values.ingress.wss.pathType | default "ImplementationSpecific" }}
+        pathType: {{ $.Values.ingress.wss.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}

From ac09b90ab6829ae91d934d0f3e93f962fb94f11a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 9 Jun 2022 07:30:33 +0200
Subject: [PATCH 08/25] ci: ensure git safe directory for test case run

---
 .github/workflows/run_test_cases.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 9a40964bc..6110c587c 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -93,6 +93,7 @@ jobs:
                 -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
                 up -d --build
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
+            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
             while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
                  != $(docker ps -a --filter name=client | wc -l) ]; do
               sleep 5

From 6b1da3bcc832e8e5021d404c05191f431471f510 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Thu, 9 Jun 2022 14:37:27 +0800
Subject: [PATCH 09/25] feat: republish support  qos & retain placeholder

---
 .../src/emqx_rule_actions.erl                 | 177 ++++++++++++++----
 .../src/emqx_rule_engine.appup.src            |  44 +++--
 2 files changed, 167 insertions(+), 54 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index edeffbde4..ea94bff22 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -35,24 +35,40 @@
             },
             target_qos => #{
                 order => 2,
-                type => number,
-                enum => [-1, 0, 1, 2],
+                type => string,
                 required => true,
-                default => 0,
+                default => <<"0">>,
                 title => #{en => <<"Target QoS">>,
                            zh => <<"目的 QoS"/utf8>>},
-                description => #{en => <<"The QoS Level to be uses when republishing the message. Set to -1 to use the original QoS">>,
-                                 zh => <<"重新发布消息时用的 QoS 级别, 设置为 -1 以使用原消息中的 QoS"/utf8>>}
+                description => #{en =>
+                                    <<"The QoS Level to be uses when republishing the message."
+                                      " Set to -1 to use the original QoS."
+                                      " Support placeholder variables.">>,
+                                 zh =>
+                                    <<"重新发布消息时用的 QoS 级别, 设置为 -1 以使用原消息中的 QoS。"
+                                      "支持占位符变量"/utf8>>}
+            },
+            target_retain => #{
+                order => 3,
+                type => string,
+                required => true,
+                default => <<"false">>,
+                title => #{en => <<"Target Retain">>,
+                           zh => <<"目标保留消息标识"/utf8>>},
+                description => #{en => <<"The Retain flag to be uses when republishing the message."
+                                         " Support placeholder variables. Default is false">>,
+                                 zh => <<"重新发布消息时用的保留消息标识，支持占位符变量。默认 false"/utf8>>}
             },
             payload_tmpl => #{
-                order => 3,
+                order => 4,
                 type => string,
                 input => textarea,
                 required => false,
                 default => <<"${payload}">>,
                 title => #{en => <<"Payload Template">>,
                            zh => <<"消息内容模板"/utf8>>},
-                description => #{en => <<"The payload template, variable interpolation is supported">>,
+                description => #{en => <<"The payload template, "
+                                         "variable interpolation is supported">>,
                                  zh => <<"消息内容模板，支持变量"/utf8>>}
             }
         }).
@@ -89,7 +105,8 @@
                params => #{},
                title => #{en => <<"Do Nothing (debug)">>,
                           zh => <<"空动作 (调试)"/utf8>>},
-               description => #{en => <<"This action does nothing and never fails. It's for debug purpose">>,
+               description => #{en => <<"This action does nothing and never fails. "
+                                        "It's for debug purpose">>,
                                 zh => <<"此动作什么都不做，并且不会失败 (用以调试)"/utf8>>}
               }).
 
@@ -113,7 +130,8 @@ on_resource_create(_Name, Conf) ->
 %%------------------------------------------------------------------------------
 %% Action 'inspect'
 %%------------------------------------------------------------------------------
--spec on_action_create_inspect(Id :: action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
+-spec on_action_create_inspect(Id :: action_instance_id(), Params :: map()) ->
+    {bindings(), NewParams :: map()}.
 on_action_create_inspect(Id, Params) ->
     Params.
 
@@ -129,12 +147,15 @@ on_action_inspect(Selected, Envs) ->
 %%------------------------------------------------------------------------------
 %% Action 'republish'
 %%------------------------------------------------------------------------------
--spec on_action_create_republish(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
+-spec on_action_create_republish(action_instance_id(), Params :: map()) ->
+    {bindings(), NewParams :: map()}.
 on_action_create_republish(Id, Params = #{
         <<"target_topic">> := TargetTopic,
-        <<"target_qos">> := TargetQoS,
+        <<"target_qos">> := TargetQoS0,
         <<"payload_tmpl">> := PayloadTmpl
        }) ->
+    {ok, TargetRetain} = to_retain(maps:get(<<"target_retain">>, Params, <<"false">>)),
+    {ok, TargetQoS} = to_qos(TargetQoS0),
     TopicTks = emqx_rule_utils:preproc_tmpl(TargetTopic),
     PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl),
     Params.
@@ -157,20 +178,28 @@ on_action_republish(Selected, _Envs = #{
                 'TargetQoS' := TargetQoS,
                 'TopicTks' := TopicTks,
                 'PayloadTks' := PayloadTks
-            }}) ->
-    ?LOG(debug, "[republish] republish to: ~p, Payload: ~p",
-        [TargetTopic, Selected]),
-    increase_and_publish(ActId,
-        #message{
-            id = emqx_guid:gen(),
-            qos = if TargetQoS =:= -1 -> QoS; true -> TargetQoS end,
-            from = ActId,
-            flags = Flags,
-            headers = #{republish_by => ActId},
-            topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
-            payload = format_msg(PayloadTks, Selected),
-            timestamp = Timestamp
-        });
+            }} = Bindings) ->
+    ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
+    TargetRetain = maps:get('TargetRetain', Bindings, false),
+    case {get_qos(TargetQoS, Selected), get_retain(TargetRetain, Selected)} of
+        {{ok, RQoS}, {ok, Retain}} when is_integer(RQoS) andalso is_boolean(Retain) ->
+            Message =
+                #message{
+                    id = emqx_guid:gen(),
+                    qos = if TargetQoS =:= -1 -> QoS; true -> RQoS end,
+                    from = ActId,
+                    flags = Flags#{retain => Retain},
+                    headers = #{republish_by => ActId},
+                    topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
+                    payload = format_msg(PayloadTks, Selected),
+                    timestamp = Timestamp
+                },
+            increase_and_publish(ActId, Message);
+        Error ->
+            emqx_rule_metrics:inc_actions_error(ActId),
+            _ = log_error(Error),
+            {badact, bad_qos_retain}
+    end;
 
 %% in case this is not a "message.publish" request
 on_action_republish(Selected, _Envs = #{
@@ -180,27 +209,43 @@ on_action_republish(Selected, _Envs = #{
                 'TargetQoS' := TargetQoS,
                 'TopicTks' := TopicTks,
                 'PayloadTks' := PayloadTks
-            }}) ->
-    ?LOG(debug, "[republish] republish to: ~p, Payload: ~p",
-        [TargetTopic, Selected]),
-    increase_and_publish(ActId,
-        #message{
-            id = emqx_guid:gen(),
-            qos = if TargetQoS =:= -1 -> 0; true -> TargetQoS end,
-            from = ActId,
-            flags = #{dup => false, retain => false},
-            headers = #{republish_by => ActId},
-            topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
-            payload = format_msg(PayloadTks, Selected),
-            timestamp = erlang:system_time(millisecond)
-        }).
+            } = Bindings}) ->
+    ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
+    TargetRetain = maps:get('TargetRetain', Bindings, false),
+    case {get_qos(TargetQoS, Selected), get_retain(TargetRetain, Selected)} of
+        {{ok, QoS}, {ok, Retain}} when is_integer(QoS) andalso is_boolean(Retain) ->
+            Message =
+                #message{
+                    id = emqx_guid:gen(),
+                    qos = QoS,
+                    from = ActId,
+                    flags = #{dup => false, retain => Retain},
+                    headers = #{republish_by => ActId},
+                    topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
+                    payload = format_msg(PayloadTks, Selected),
+                    timestamp = erlang:system_time(millisecond)
+                },
+            increase_and_publish(ActId, Message);
+        Error ->
+            emqx_rule_metrics:inc_actions_error(ActId),
+            _ = log_error(Error),
+            {badact, bad_qos_retain}
+    end.
+
+log_error({{ok, _}, RetainError}) ->
+    ?LOG(error, "[republish] invalid retain: ~p", [RetainError]);
+log_error({QosError, {ok, _}}) ->
+    ?LOG(error, "[republish] invalid qos: ~p", [QosError]);
+log_error({QosError, RetainError}) ->
+    ?LOG(error, "[republish] invalid qos: ~p invalid retain: ~p", [QosError, RetainError]).
 
 increase_and_publish(ActId, Msg) ->
     _ = emqx_broker:safe_publish(Msg),
     emqx_rule_metrics:inc_actions_success(ActId),
     emqx_metrics:inc_msg(Msg).
 
--spec on_action_create_do_nothing(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
+-spec on_action_create_do_nothing(action_instance_id(), Params :: map()) ->
+    {bindings(), NewParams :: map()}.
 on_action_create_do_nothing(ActId, Params) when is_binary(ActId) ->
     Params.
 
@@ -211,3 +256,55 @@ format_msg([], Data) ->
     emqx_json:encode(Data);
 format_msg(Tokens, Data) ->
      emqx_rule_utils:proc_tmpl(Tokens, Data).
+
+get_qos(-1, _Data) -> {ok, 0};
+get_qos(0, _Data) ->  {ok, 0};
+get_qos(1, _Data) ->  {ok, 1};
+get_qos(2, _Data) ->  {ok, 2};
+get_qos({path, Path}, Data) ->
+    to_qos(emqx_rule_maps:nested_get({path, Path}, Data, 0)).
+
+to_qos(0) ->        {ok, 0};
+to_qos(1) ->        {ok, 1};
+to_qos(2) ->        {ok, 2};
+to_qos(<<"-1">>) -> {ok, 0};
+to_qos(<<"0">>) ->  {ok, 0};
+to_qos(<<"1">>) ->  {ok, 1};
+to_qos(<<"2">>) ->  {ok, 2};
+to_qos(TargetQoS) ->
+    case parse_value_or_placeholder(TargetQoS) of
+        {path, P} ->
+            {ok, {path, P}};
+        _ ->
+            {error, bad_qos}
+    end.
+
+get_retain(false, _Data) -> {ok, false};
+get_retain(true, _Data) ->  {ok, true};
+get_retain({path, Path}, Data) ->
+    to_retain(emqx_rule_maps:nested_get({path, Path}, Data, true)).
+
+to_retain(true) ->        {ok, true};
+to_retain(false) ->       {ok, false};
+to_retain(<<"true">>) ->  {ok, true};
+to_retain(<<"false">>) -> {ok, false};
+to_retain(<<"1">>) ->     {ok, true};
+to_retain(<<"0">>) ->     {ok, false};
+to_retain(1) ->           {ok, true};
+to_retain(0) ->           {ok, false};
+to_retain(TargetRetain) ->
+    case parse_value_or_placeholder(TargetRetain) of
+        {path, P} ->
+            {ok, {path, P}};
+        _ ->
+            {error, bad_retain}
+    end.
+
+parse_value_or_placeholder(ValueOrPlaceholder) ->
+    case re:run(ValueOrPlaceholder, "^\\$\{.+\}$") of
+        nomatch ->
+            ValueOrPlaceholder;
+        {match, [{0, Length}]} ->
+            Placeholder = binary:part(ValueOrPlaceholder, 2, Length - 3),
+            {path, [{key, Key} || Key <- string:lexemes(Placeholder, ". ")]}
+    end.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 7b4049254..5202b88b3 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,9 +1,12 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.10",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.10",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
@@ -12,7 +15,8 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -25,7 +29,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -38,7 +43,8 @@
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -51,7 +57,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -64,7 +71,8 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -136,9 +144,12 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.10",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.10",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
@@ -147,7 +158,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.8",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -160,7 +172,8 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.7",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -173,7 +186,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.6",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -186,7 +200,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.5",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -199,7 +214,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.4",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},

From ac700b8e6f1d05058e55e411d0da5b8addd6b433 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Thu, 9 Jun 2022 17:35:55 +0800
Subject: [PATCH 10/25] fix(rule): replubish SUIT

---
 .../test/emqx_rule_engine_SUITE.erl           | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 6263c8440..f5c379159 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -413,7 +413,8 @@ t_republish_action(_Config) ->
                     #{rawsql => <<"select topic, payload, qos from \"t1\"">>,
                       actions => [#{name => 'republish',
                                     args => #{<<"target_topic">> => <<"t2">>,
-                                              <<"target_qos">> => -1,
+                                              <<"target_qos">> => <<"-1">>,
+                                              <<"target_retain">> => <<"false">>,
                                               <<"payload_tmpl">> => <<"${payload}">>}}],
                       description => <<"builtin-republish-rule">>}),
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
@@ -479,7 +480,8 @@ t_crud_rule_api(_Config) ->
                         {<<"params">>,[
                             {<<"arg1">>,1},
                             {<<"target_topic">>, <<"t2">>},
-                            {<<"target_qos">>, -1},
+                            {<<"target_qos">>, <<"0">>},
+                            {<<"target_retain">>, <<"false">>},
                             {<<"payload_tmpl">>, <<"${payload}">>}
                         ]}
                     ]]
@@ -1618,7 +1620,8 @@ t_sqlselect_multi_actoins_1(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                      },
                             fallbacks => []}
@@ -1644,7 +1647,8 @@ t_sqlselect_multi_actoins_1_1(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1672,7 +1676,8 @@ t_sqlselect_multi_actoins_2(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1703,7 +1708,8 @@ t_sqlselect_multi_actoins_3(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1741,7 +1747,8 @@ t_sqlselect_multi_actoins_3_1(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1780,7 +1787,8 @@ t_sqlselect_multi_actoins_4(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => -1,
+                                      <<"target_qos">> => <<"-1">>,
+                                      <<"target_retain">> => <<"false">>,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -2534,7 +2542,8 @@ create_simple_repub_rule(TargetTopic, SQL, Template) ->
                     #{rawsql => SQL,
                       actions => [#{name => 'republish',
                                     args => #{<<"target_topic">> => TargetTopic,
-                                              <<"target_qos">> => -1,
+                                              <<"target_qos">> => <<"-1">>,
+                                              <<"target_retain">> => <<"false">>,
                                               <<"payload_tmpl">> => Template}
                                     }],
                       description => <<"simple repub rule">>}),

From 34d84599c3440525dab49fb5dde5c24158c414f9 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 9 Jun 2022 11:34:58 -0300
Subject: [PATCH 11/25] ci(appup): upload appup files when ci check fails

Often we have to remember to update both local tags and local
dependencies, and even so local results might differ from what the CI
check expects.  To help fixing expected appup file differences, we can
upload what CI produced to apply to our branch.
---
 .github/workflows/apps_version_check.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml
index 59d6aebff..c36ceea2a 100644
--- a/.github/workflows/apps_version_check.yaml
+++ b/.github/workflows/apps_version_check.yaml
@@ -29,3 +29,10 @@ jobs:
         run: ./scripts/update-appup.sh emqx-ee --check
       - name: Check apps version
         run: ./scripts/apps-version-check.sh
+      - uses: actions/upload-artifact@v3.1.0
+        if: failure()
+        with:
+          name: expected_appup_files
+          path: |
+            {src,apps}/**/*.appup.src
+          retention-days: 1

From 6a5c896f56602e46bc421057b48de9c28594754e Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 9 Jun 2022 11:51:02 -0300
Subject: [PATCH 12/25] ci(relup): fix: follow URL redirects when checking for
 package existence

---
 scripts/relup-base-packages.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh
index bedc050c7..1c99f0c9c 100755
--- a/scripts/relup-base-packages.sh
+++ b/scripts/relup-base-packages.sh
@@ -57,7 +57,7 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
     filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip"
     url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename"
     echo "downloading base package from ${url} ..."
-    if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+    if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
         curl -L -o "${filename}" "${url}"
         if [ "$SYSTEM" != "centos6" ]; then
             curl -L -o "${filename}.sha256" "${url}.sha256"

From 7d848950c77eb598d46990849644b8c559c2c46b Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Fri, 10 Jun 2022 11:37:34 +0800
Subject: [PATCH 13/25] fix(republish): to_qos & to_retain, add new util funcs

---
 .../src/emqx_rule_actions.erl                 | 109 +++++++++---------
 .../src/emqx_rule_engine.appup.src            |  68 +++++++----
 apps/emqx_rule_engine/src/emqx_rule_utils.erl |  20 ++++
 .../src/emqx_rule_validator.erl               |   4 +
 4 files changed, 126 insertions(+), 75 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index ea94bff22..1ecf134cb 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -35,23 +35,27 @@
             },
             target_qos => #{
                 order => 2,
-                type => string,
+                type => editable_select,
+                enum => [<<"0">>, <<"1">>, <<"2">>],
                 required => true,
                 default => <<"0">>,
                 title => #{en => <<"Target QoS">>,
                            zh => <<"目的 QoS"/utf8>>},
                 description => #{en =>
                                     <<"The QoS Level to be uses when republishing the message."
-                                      " Set to -1 to use the original QoS."
-                                      " Support placeholder variables.">>,
+                                      " Support placeholder variables."
+                                      " Set to ${qos} to use the original QoS."
+                                      " Or other variable, value is 0 or 1 or 2">>,
                                  zh =>
-                                    <<"重新发布消息时用的 QoS 级别, 设置为 -1 以使用原消息中的 QoS。"
-                                      "支持占位符变量"/utf8>>}
+                                    <<"重新发布消息时用的 QoS 级别。"
+                                      "支持占位符变量，可以使用 ${qos} 来使用原消息的 QoS，"
+                                      "或其他值为 0 或 1 或 2 的变量。"/utf8>>}
             },
             target_retain => #{
                 order => 3,
-                type => string,
-                required => true,
+                type => editable_select,
+                enum => [<<"true">>, <<"false">>],
+                required => false,
                 default => <<"false">>,
                 title => #{en => <<"Target Retain">>,
                            zh => <<"目标保留消息标识"/utf8>>},
@@ -226,9 +230,9 @@ on_action_republish(Selected, _Envs = #{
                     timestamp = erlang:system_time(millisecond)
                 },
             increase_and_publish(ActId, Message);
-        Error ->
+        Errors ->
             emqx_rule_metrics:inc_actions_error(ActId),
-            _ = log_error(Error),
+            _ = log_error(Errors),
             {badact, bad_qos_retain}
     end.
 
@@ -257,54 +261,53 @@ format_msg([], Data) ->
 format_msg(Tokens, Data) ->
      emqx_rule_utils:proc_tmpl(Tokens, Data).
 
-get_qos(-1, _Data) -> {ok, 0};
-get_qos(0, _Data) ->  {ok, 0};
-get_qos(1, _Data) ->  {ok, 1};
-get_qos(2, _Data) ->  {ok, 2};
-get_qos({path, Path}, Data) ->
-    to_qos(emqx_rule_maps:nested_get({path, Path}, Data, 0)).
-
-to_qos(0) ->        {ok, 0};
-to_qos(1) ->        {ok, 1};
-to_qos(2) ->        {ok, 2};
-to_qos(<<"-1">>) -> {ok, 0};
-to_qos(<<"0">>) ->  {ok, 0};
-to_qos(<<"1">>) ->  {ok, 1};
-to_qos(<<"2">>) ->  {ok, 2};
 to_qos(TargetQoS) ->
-    case parse_value_or_placeholder(TargetQoS) of
-        {path, P} ->
-            {ok, {path, P}};
-        _ ->
-            {error, bad_qos}
+    case get_qos(TargetQoS) of
+        {ok, QoS} ->
+            {ok, QoS};
+        _Error ->
+            case emqx_rule_utils:preproc_tmpl(TargetQoS) of
+                Tmpl = [{var, _}] ->
+                    {ok, Tmpl};
+                _ ->
+                    {error, bad_qos}
+            end
     end.
 
-get_retain(false, _Data) -> {ok, false};
-get_retain(true, _Data) ->  {ok, true};
-get_retain({path, Path}, Data) ->
-    to_retain(emqx_rule_maps:nested_get({path, Path}, Data, true)).
+get_qos(Tmpl, Data) ->
+    get_qos(emqx_rule_utils:replace_simple_var(Tmpl, Data)).
+
+get_qos(<<"-1">>) -> {ok, 0};
+get_qos(<<"0">>) ->  {ok, 0};
+get_qos(<<"1">>) ->  {ok, 1};
+get_qos(<<"2">>) ->  {ok, 2};
+get_qos(0) ->        {ok, 0};
+get_qos(1) ->        {ok, 1};
+get_qos(2) ->        {ok, 2};
+get_qos(_) ->        {error, bad_qos}.
 
-to_retain(true) ->        {ok, true};
-to_retain(false) ->       {ok, false};
-to_retain(<<"true">>) ->  {ok, true};
-to_retain(<<"false">>) -> {ok, false};
-to_retain(<<"1">>) ->     {ok, true};
-to_retain(<<"0">>) ->     {ok, false};
-to_retain(1) ->           {ok, true};
-to_retain(0) ->           {ok, false};
 to_retain(TargetRetain) ->
-    case parse_value_or_placeholder(TargetRetain) of
-        {path, P} ->
-            {ok, {path, P}};
-        _ ->
-            {error, bad_retain}
+    case get_retain(TargetRetain) of
+        {ok, Retain} ->
+            {ok, Retain};
+        _Error ->
+            case emqx_rule_utils:preproc_tmpl(TargetRetain) of
+                Tmpl = [{var, _}] ->
+                    {ok, Tmpl};
+                _ ->
+                    {error, bad_retain}
+            end
     end.
 
-parse_value_or_placeholder(ValueOrPlaceholder) ->
-    case re:run(ValueOrPlaceholder, "^\\$\{.+\}$") of
-        nomatch ->
-            ValueOrPlaceholder;
-        {match, [{0, Length}]} ->
-            Placeholder = binary:part(ValueOrPlaceholder, 2, Length - 3),
-            {path, [{key, Key} || Key <- string:lexemes(Placeholder, ". ")]}
-    end.
+get_retain(Tmpl, Data) ->
+    get_retain(emqx_rule_utils:replace_simple_var(Tmpl, Data)).
+
+get_retain(true) ->        {ok, true};
+get_retain(false) ->       {ok, false};
+get_retain(<<"true">>) ->  {ok, true};
+get_retain(<<"false">>) -> {ok, false};
+get_retain(<<"1">>) ->     {ok, true};
+get_retain(<<"0">>) ->     {ok, false};
+get_retain(1) ->           {ok, true};
+get_retain(0) ->           {ok, false};
+get_retain(_) ->           {error, bad_retain}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 5202b88b3..b92d67e4f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -2,10 +2,13 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.10",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -15,7 +18,8 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
@@ -29,7 +33,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -43,7 +48,8 @@
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -57,7 +63,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -71,7 +78,8 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -85,7 +93,8 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -99,7 +108,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -114,7 +124,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -129,7 +140,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
-    [{add_module,emqx_rule_date},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -145,10 +157,13 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.10",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -158,7 +173,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.8",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
@@ -172,7 +188,8 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.7",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -186,7 +203,8 @@
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.6",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -200,7 +218,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.5",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -214,7 +233,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.4",
-    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
@@ -228,7 +248,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.3",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -242,7 +263,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.2",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -257,7 +279,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.1",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -272,7 +295,8 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {delete_module,emqx_rule_date}]},
    {"4.3.0",
-    [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
index d287f1ad0..047b9d1af 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
@@ -16,6 +16,10 @@
 
 -module(emqx_rule_utils).
 
+-export([ replace_vars_in_str/2
+        , replace_simple_var/2
+        ]).
+
 %% preprocess and process tempalte string with place holders
 -export([ preproc_tmpl/1
         , proc_tmpl/2
@@ -87,6 +91,22 @@ preproc_tmpl([[Str, Phld]| Tokens], Acc) ->
 preproc_tmpl([[Str]| Tokens], Acc) ->
     preproc_tmpl(Tokens, put_head(str, Str, Acc)).
 
+%% Replace a string contains vars to another string in which the placeholders are replace by the
+%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
+%% "a: 1".
+replace_vars_in_str(Tokens, Data) when is_list(Tokens) ->
+    proc_tmpl(Tokens, Data, #{return => full_binary});
+replace_vars_in_str(Val, _Data) ->
+    Val.
+
+%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result
+%% value will be an integer 1.
+replace_simple_var(Tokens, Data) when is_list(Tokens) ->
+    [Var] = proc_tmpl(Tokens, Data, #{return => rawlist}),
+    Var;
+replace_simple_var(Val, _Data) ->
+    Val.
+
 put_head(_Type, <<>>, List) -> List;
 put_head(Type, Term, List) ->
     [{Type, Term} | List].
diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl
index e32ec66ab..57d5eb465 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_validator.erl
@@ -47,6 +47,7 @@
         , array
         , file
         , cfgselect %% TODO: [5.0] refactor this
+        , editable_select
         ]).
 
 %%------------------------------------------------------------------------------
@@ -84,6 +85,9 @@ validate_spec(ParamsSepc) ->
 %% Internal Functions
 %%------------------------------------------------------------------------------
 
+%% Validate editable_select first, because editable_select has enum selection.
+validate_value(Val, #{type := editable_select}) ->
+    Val;
 validate_value(Val, #{enum := Enum}) ->
     validate_enum(Val, Enum);
 validate_value(Val, #{type := object} = Spec) ->

From e1e2fd50fd4f142c9820e50083c315b9b9682881 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Fri, 10 Jun 2022 16:08:33 +0800
Subject: [PATCH 14/25] fix(republish): action input editable_select

---
 .../src/emqx_rule_actions.erl                 | 181 ++++++++----------
 .../src/emqx_rule_validator.erl               |  18 +-
 2 files changed, 98 insertions(+), 101 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index 1ecf134cb..aa5739da2 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -35,10 +35,11 @@
             },
             target_qos => #{
                 order => 2,
-                type => editable_select,
-                enum => [<<"0">>, <<"1">>, <<"2">>],
+                input => editable_select,
+                type => [number, string],
+                enum => [0, 1, 2, <<"${qos}">>],
                 required => true,
-                default => <<"0">>,
+                default => 0,
                 title => #{en => <<"Target QoS">>,
                            zh => <<"目的 QoS"/utf8>>},
                 description => #{en =>
@@ -48,20 +49,24 @@
                                       " Or other variable, value is 0 or 1 or 2">>,
                                  zh =>
                                     <<"重新发布消息时用的 QoS 级别。"
-                                      "支持占位符变量，可以使用 ${qos} 来使用原消息的 QoS，"
+                                      "支持占位符变量，可以填写 ${qos} 来使用原消息的 QoS，"
                                       "或其他值为 0 或 1 或 2 的变量。"/utf8>>}
             },
             target_retain => #{
                 order => 3,
-                type => editable_select,
-                enum => [<<"true">>, <<"false">>],
+                input => editable_select,
+                type => [boolean, string],
+                enum => [true, false, <<"${flags.retain}">>],
                 required => false,
-                default => <<"false">>,
+                default => false,
                 title => #{en => <<"Target Retain">>,
                            zh => <<"目标保留消息标识"/utf8>>},
                 description => #{en => <<"The Retain flag to be uses when republishing the message."
+                                         " Set to ${flags.retain} to use the original Retain."
                                          " Support placeholder variables. Default is false">>,
-                                 zh => <<"重新发布消息时用的保留消息标识，支持占位符变量。默认 false"/utf8>>}
+                                 zh => <<"重新发布消息时用的保留消息标识。"
+                                         "支持占位符变量，可以填写 ${flags.retain} 来使用原消息的 Retain。"
+                                         "默认 false"/utf8>>}
             },
             payload_tmpl => #{
                 order => 4,
@@ -158,8 +163,8 @@ on_action_create_republish(Id, Params = #{
         <<"target_qos">> := TargetQoS0,
         <<"payload_tmpl">> := PayloadTmpl
        }) ->
-    {ok, TargetRetain} = to_retain(maps:get(<<"target_retain">>, Params, <<"false">>)),
-    {ok, TargetQoS} = to_qos(TargetQoS0),
+    TargetRetain = to_retain(maps:get(<<"target_retain">>, Params, <<"false">>)),
+    TargetQoS = to_qos(TargetQoS0),
     TopicTks = emqx_rule_utils:preproc_tmpl(TargetTopic),
     PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl),
     Params.
@@ -182,28 +187,21 @@ on_action_republish(Selected, _Envs = #{
                 'TargetQoS' := TargetQoS,
                 'TopicTks' := TopicTks,
                 'PayloadTks' := PayloadTks
-            }} = Bindings) ->
+            } = Bindings}) ->
     ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
     TargetRetain = maps:get('TargetRetain', Bindings, false),
-    case {get_qos(TargetQoS, Selected), get_retain(TargetRetain, Selected)} of
-        {{ok, RQoS}, {ok, Retain}} when is_integer(RQoS) andalso is_boolean(Retain) ->
-            Message =
-                #message{
-                    id = emqx_guid:gen(),
-                    qos = if TargetQoS =:= -1 -> QoS; true -> RQoS end,
-                    from = ActId,
-                    flags = Flags#{retain => Retain},
-                    headers = #{republish_by => ActId},
-                    topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
-                    payload = format_msg(PayloadTks, Selected),
-                    timestamp = Timestamp
-                },
-            increase_and_publish(ActId, Message);
-        Error ->
-            emqx_rule_metrics:inc_actions_error(ActId),
-            _ = log_error(Error),
-            {badact, bad_qos_retain}
-    end;
+    Message =
+        #message{
+            id = emqx_guid:gen(),
+            qos = get_qos(TargetQoS, Selected, QoS),
+            from = ActId,
+            flags = Flags#{retain => get_retain(TargetRetain, Selected)},
+            headers = #{republish_by => ActId},
+            topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
+            payload = format_msg(PayloadTks, Selected),
+            timestamp = Timestamp
+        },
+    increase_and_publish(ActId, Message);
 
 %% in case this is not a "message.publish" request
 on_action_republish(Selected, _Envs = #{
@@ -216,32 +214,18 @@ on_action_republish(Selected, _Envs = #{
             } = Bindings}) ->
     ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
     TargetRetain = maps:get('TargetRetain', Bindings, false),
-    case {get_qos(TargetQoS, Selected), get_retain(TargetRetain, Selected)} of
-        {{ok, QoS}, {ok, Retain}} when is_integer(QoS) andalso is_boolean(Retain) ->
-            Message =
-                #message{
-                    id = emqx_guid:gen(),
-                    qos = QoS,
-                    from = ActId,
-                    flags = #{dup => false, retain => Retain},
-                    headers = #{republish_by => ActId},
-                    topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
-                    payload = format_msg(PayloadTks, Selected),
-                    timestamp = erlang:system_time(millisecond)
-                },
-            increase_and_publish(ActId, Message);
-        Errors ->
-            emqx_rule_metrics:inc_actions_error(ActId),
-            _ = log_error(Errors),
-            {badact, bad_qos_retain}
-    end.
-
-log_error({{ok, _}, RetainError}) ->
-    ?LOG(error, "[republish] invalid retain: ~p", [RetainError]);
-log_error({QosError, {ok, _}}) ->
-    ?LOG(error, "[republish] invalid qos: ~p", [QosError]);
-log_error({QosError, RetainError}) ->
-    ?LOG(error, "[republish] invalid qos: ~p invalid retain: ~p", [QosError, RetainError]).
+    Message =
+        #message{
+            id = emqx_guid:gen(),
+            qos = get_qos(TargetQoS, Selected, 0),
+            from = ActId,
+            flags = #{dup => false, retain => get_retain(TargetRetain, Selected)},
+            headers = #{republish_by => ActId},
+            topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected),
+            payload = format_msg(PayloadTks, Selected),
+            timestamp = erlang:system_time(millisecond)
+        },
+    increase_and_publish(ActId, Message).
 
 increase_and_publish(ActId, Msg) ->
     _ = emqx_broker:safe_publish(Msg),
@@ -261,53 +245,56 @@ format_msg([], Data) ->
 format_msg(Tokens, Data) ->
      emqx_rule_utils:proc_tmpl(Tokens, Data).
 
+%% -1 for old version.
+to_qos(<<"-1">>) -> -1;
+to_qos(-1) -> -1;
 to_qos(TargetQoS) ->
-    case get_qos(TargetQoS) of
-        {ok, QoS} ->
-            {ok, QoS};
-        _Error ->
-            case emqx_rule_utils:preproc_tmpl(TargetQoS) of
-                Tmpl = [{var, _}] ->
-                    {ok, Tmpl};
-                _ ->
-                    {error, bad_qos}
-            end
+    try
+        qos(TargetQoS)
+    catch _:_ ->
+        %% Use placeholder.
+        case emqx_rule_utils:preproc_tmpl(TargetQoS) of
+            Tmpl = [{var, _}] ->
+                Tmpl;
+            _BadQoS ->
+                error({bad_qos, TargetQoS})
+        end
     end.
 
-get_qos(Tmpl, Data) ->
-    get_qos(emqx_rule_utils:replace_simple_var(Tmpl, Data)).
+get_qos(-1, _Data, Default) -> Default;
+get_qos(TargetQoS, Data, _Default) ->
+    qos(emqx_rule_utils:replace_simple_var(TargetQoS, Data)).
 
-get_qos(<<"-1">>) -> {ok, 0};
-get_qos(<<"0">>) ->  {ok, 0};
-get_qos(<<"1">>) ->  {ok, 1};
-get_qos(<<"2">>) ->  {ok, 2};
-get_qos(0) ->        {ok, 0};
-get_qos(1) ->        {ok, 1};
-get_qos(2) ->        {ok, 2};
-get_qos(_) ->        {error, bad_qos}.
+qos(<<"0">>) ->  0;
+qos(<<"1">>) ->  1;
+qos(<<"2">>) ->  2;
+qos(0) ->        0;
+qos(1) ->        1;
+qos(2) ->        2;
+qos(BadQoS) -> error({bad_qos, BadQoS}).
 
 to_retain(TargetRetain) ->
-    case get_retain(TargetRetain) of
-        {ok, Retain} ->
-            {ok, Retain};
-        _Error ->
-            case emqx_rule_utils:preproc_tmpl(TargetRetain) of
-                Tmpl = [{var, _}] ->
-                    {ok, Tmpl};
-                _ ->
-                    {error, bad_retain}
-            end
+    try
+        retain(TargetRetain)
+    catch _:_ ->
+        %% Use placeholder.
+        case emqx_rule_utils:preproc_tmpl(TargetRetain) of
+            Tmpl = [{var, _}] ->
+                Tmpl;
+            _BadRetain ->
+                error({bad_retain, TargetRetain})
+        end
     end.
 
-get_retain(Tmpl, Data) ->
-    get_retain(emqx_rule_utils:replace_simple_var(Tmpl, Data)).
+get_retain(TargetRetain, Data) ->
+    retain(emqx_rule_utils:replace_simple_var(TargetRetain, Data)).
 
-get_retain(true) ->        {ok, true};
-get_retain(false) ->       {ok, false};
-get_retain(<<"true">>) ->  {ok, true};
-get_retain(<<"false">>) -> {ok, false};
-get_retain(<<"1">>) ->     {ok, true};
-get_retain(<<"0">>) ->     {ok, false};
-get_retain(1) ->           {ok, true};
-get_retain(0) ->           {ok, false};
-get_retain(_) ->           {error, bad_retain}.
+retain(true) ->        true;
+retain(false) ->       false;
+retain(<<"true">>) ->  true;
+retain(<<"false">>) -> false;
+retain(<<"1">>) ->     true;
+retain(<<"0">>) ->     false;
+retain(1) ->           true;
+retain(0) ->           false;
+retain(BadRetain) -> error({bad_retain, BadRetain}).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl
index 57d5eb465..237e5dfba 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_validator.erl
@@ -47,7 +47,6 @@
         , array
         , file
         , cfgselect %% TODO: [5.0] refactor this
-        , editable_select
         ]).
 
 %%------------------------------------------------------------------------------
@@ -85,9 +84,8 @@ validate_spec(ParamsSepc) ->
 %% Internal Functions
 %%------------------------------------------------------------------------------
 
-%% Validate editable_select first, because editable_select has enum selection.
-validate_value(Val, #{type := editable_select}) ->
-    Val;
+validate_value(Val, #{type := Types} = Spec) when is_list(Types) ->
+    validate_types(Val, Types, Spec);
 validate_value(Val, #{enum := Enum}) ->
     validate_enum(Val, Enum);
 validate_value(Val, #{type := object} = Spec) ->
@@ -95,6 +93,15 @@ validate_value(Val, #{type := object} = Spec) ->
 validate_value(Val, #{type := Type} = Spec) ->
     validate_type(Val, Type, Spec).
 
+validate_types(Val, [], _Spec) ->
+    throw({invalid_data_type, Val});
+validate_types(Val, [Type | Types], Spec) ->
+    try
+        validate_type(Val, Type, Spec)
+    catch _:_ ->
+        validate_types(Val, Types, Spec)
+    end.
+
 validate_type(Val, file, _Spec) ->
     validate_file(Val);
 validate_type(Val, String, Spec) when String =:= string;
@@ -161,6 +168,9 @@ do_validate_spec(Name, #{type := array} = Spec) ->
         fun (not_found) -> throw({required_field_missing, {items, {in, Name}}});
             (Items) -> do_validate_spec(Name, Items)
         end);
+do_validate_spec(_Name, #{type := Types}) when is_list(Types) ->
+    _ = [ok = supported_data_type(Type, ?DATA_TYPES) || Type <- Types],
+    ok;
 do_validate_spec(_Name, #{type := Type}) ->
     _ = supported_data_type(Type, ?DATA_TYPES);
 

From c5f754c3b88c47fe7e53fd2a841b05cb9c53cace Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Fri, 10 Jun 2022 17:16:14 +0800
Subject: [PATCH 15/25] fix(suite): add republish qos & retain test case

---
 .../src/emqx_rule_actions.erl                 | 10 ++--
 .../test/emqx_rule_engine_SUITE.erl           | 51 +++++++++++--------
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index aa5739da2..83c13a3af 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -43,14 +43,12 @@
                 title => #{en => <<"Target QoS">>,
                            zh => <<"目的 QoS"/utf8>>},
                 description => #{en =>
-                                    <<"The QoS Level to be uses when republishing the message."
+                                    <<"The QoS Level to be used when republishing the message."
                                       " Support placeholder variables."
-                                      " Set to ${qos} to use the original QoS."
-                                      " Or other variable, value is 0 or 1 or 2">>,
+                                      " Set to ${qos} to use the original QoS. Default is 0">>,
                                  zh =>
                                     <<"重新发布消息时用的 QoS 级别。"
-                                      "支持占位符变量，可以填写 ${qos} 来使用原消息的 QoS，"
-                                      "或其他值为 0 或 1 或 2 的变量。"/utf8>>}
+                                      "支持占位符变量，可以填写 ${qos} 来使用原消息的 QoS。默认 0"/utf8>>}
             },
             target_retain => #{
                 order => 3,
@@ -61,7 +59,7 @@
                 default => false,
                 title => #{en => <<"Target Retain">>,
                            zh => <<"目标保留消息标识"/utf8>>},
-                description => #{en => <<"The Retain flag to be uses when republishing the message."
+                description => #{en => <<"The Retain flag to be used when republishing the message."
                                          " Set to ${flags.retain} to use the original Retain."
                                          " Support placeholder variables. Default is false">>,
                                  zh => <<"重新发布消息时用的保留消息标识。"
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index f5c379159..663eddd5a 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -405,16 +405,31 @@ t_reset_metrics(_Config) ->
     ok.
 
 t_republish_action(_Config) ->
-    Qos0Received = emqx_metrics:val('messages.qos0.received'),
+    TargetQoSList = [-1, 0, 1, 2, <<"${qos}">>],
+    TargetRetainList = [true, false, <<"${flags.retain}">>],
+    [[republish_action_test(TargetQoS, TargetRetain) || TargetRetain <- TargetRetainList]
+        || TargetQoS <- TargetQoSList],
+    ok.
+
+republish_action_test(TargetQoS, TargetRetain) ->
+    {QoSReceivedMetricsName, PubQoS} =
+        case TargetQoS of
+            <<"${qos}">> -> {'messages.qos0.received', 0};
+            -1 -> {'messages.qos0.received', 0};
+            0 -> {'messages.qos0.received', 0};
+            1 -> {'messages.qos1.received', 1};
+            2 -> {'messages.qos2.received', 2}
+        end,
+    QosReceived = emqx_metrics:val(QoSReceivedMetricsName),
     Received = emqx_metrics:val('messages.received'),
     ok = emqx_rule_engine:load_providers(),
     {ok, #rule{id = Id, for = [<<"t1">>]}} =
         emqx_rule_engine:create_rule(
-                    #{rawsql => <<"select topic, payload, qos from \"t1\"">>,
+                    #{rawsql => <<"select * from \"t1\"">>,
                       actions => [#{name => 'republish',
                                     args => #{<<"target_topic">> => <<"t2">>,
-                                              <<"target_qos">> => <<"-1">>,
-                                              <<"target_retain">> => <<"false">>,
+                                              <<"target_qos">> => TargetQoS,
+                                              <<"target_retain">> => TargetRetain,
                                               <<"payload_tmpl">> => <<"${payload}">>}}],
                       description => <<"builtin-republish-rule">>}),
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
@@ -422,7 +437,7 @@ t_republish_action(_Config) ->
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
 
     Msg = <<"{\"id\": 1, \"name\": \"ha\"}">>,
-    emqtt:publish(Client, <<"t1">>, Msg, 0),
+    emqtt:publish(Client, <<"t1">>, Msg, PubQoS),
     receive {publish, #{topic := <<"t2">>, payload := Payload}} ->
         ?assertEqual(Msg, Payload)
     after 1000 ->
@@ -430,7 +445,7 @@ t_republish_action(_Config) ->
     end,
     emqtt:stop(Client),
     emqx_rule_registry:remove_rule(Id),
-    ?assertEqual(2, emqx_metrics:val('messages.qos0.received') - Qos0Received),
+    ?assertEqual(2, emqx_metrics:val(QoSReceivedMetricsName) - QosReceived),
     ?assertEqual(2, emqx_metrics:val('messages.received') - Received),
     ok.
 
@@ -480,8 +495,7 @@ t_crud_rule_api(_Config) ->
                         {<<"params">>,[
                             {<<"arg1">>,1},
                             {<<"target_topic">>, <<"t2">>},
-                            {<<"target_qos">>, <<"0">>},
-                            {<<"target_retain">>, <<"false">>},
+                            {<<"target_qos">>, 0},
                             {<<"payload_tmpl">>, <<"${payload}">>}
                         ]}
                     ]]
@@ -1620,8 +1634,7 @@ t_sqlselect_multi_actoins_1(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                      },
                             fallbacks => []}
@@ -1647,8 +1660,7 @@ t_sqlselect_multi_actoins_1_1(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1676,8 +1688,7 @@ t_sqlselect_multi_actoins_2(Config) ->
                           #{name => 'crash_action', args => #{}, fallbacks => []},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1708,8 +1719,7 @@ t_sqlselect_multi_actoins_3(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1747,8 +1757,7 @@ t_sqlselect_multi_actoins_3_1(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -1787,8 +1796,7 @@ t_sqlselect_multi_actoins_4(Config) ->
                           ]},
                           #{name => 'republish',
                             args => #{<<"target_topic">> => <<"t2">>,
-                                      <<"target_qos">> => <<"-1">>,
-                                      <<"target_retain">> => <<"false">>,
+                                      <<"target_qos">> => -1,
                                       <<"payload_tmpl">> => <<"clientid=${clientid}">>
                                     },
                             fallbacks => []}
@@ -2542,8 +2550,7 @@ create_simple_repub_rule(TargetTopic, SQL, Template) ->
                     #{rawsql => SQL,
                       actions => [#{name => 'republish',
                                     args => #{<<"target_topic">> => TargetTopic,
-                                              <<"target_qos">> => <<"-1">>,
-                                              <<"target_retain">> => <<"false">>,
+                                              <<"target_qos">> => -1,
                                               <<"payload_tmpl">> => Template}
                                     }],
                       description => <<"simple repub rule">>}),

From 1ba8ad4c25d4f66f00a72e190a3bd56f3aa37271 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 10 Jun 2022 15:00:57 -0300
Subject: [PATCH 16/25] fix(metrics): inc `connack.auth_error` when using MQTT
 3.1

Since MQTT 3.1 uses a different reason code for auth failures, it was
failing to increase the corresponding metric that works for MQTT 5.0.
---
 src/emqx_metrics.erl       |  8 ++++++--
 test/emqx_broker_SUITE.erl | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl
index 7bec83465..c919e25eb 100644
--- a/src/emqx_metrics.erl
+++ b/src/emqx_metrics.erl
@@ -429,8 +429,12 @@ inc_sent(Packet) ->
 
 do_inc_sent(?CONNACK_PACKET(ReasonCode)) ->
     (ReasonCode == ?RC_SUCCESS) orelse inc('packets.connack.error'),
-    (ReasonCode == ?RC_NOT_AUTHORIZED) andalso inc('packets.connack.auth_error'),
-    (ReasonCode == ?RC_BAD_USER_NAME_OR_PASSWORD) andalso inc('packets.connack.auth_error'),
+    ((ReasonCode == ?RC_NOT_AUTHORIZED)
+     orelse (ReasonCode == ?CONNACK_AUTH))
+        andalso inc('packets.connack.auth_error'),
+    ((ReasonCode == ?RC_BAD_USER_NAME_OR_PASSWORD)
+     orelse (ReasonCode == ?CONNACK_CREDENTIALS))
+        andalso inc('packets.connack.auth_error'),
     inc('packets.connack.sent');
 
 do_inc_sent(?PUBLISH_PACKET(QoS)) ->
diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl
index e8d19c5c0..ec46ff8e1 100644
--- a/test/emqx_broker_SUITE.erl
+++ b/test/emqx_broker_SUITE.erl
@@ -277,6 +277,38 @@ t_stats_fun({'end', _Config}) ->
     ok = emqx_broker:unsubscribe(<<"topic">>),
     ok = emqx_broker:unsubscribe(<<"topic2">>).
 
+t_connack_auth_error({init, Config}) ->
+    process_flag(trap_exit, true),
+    emqx_ct_helpers:stop_apps([]),
+    emqx_ct_helpers:boot_modules(all),
+    Handler =
+        fun(emqx) ->
+                application:set_env(emqx, acl_nomatch, deny),
+                application:set_env(emqx, allow_anonymous, false),
+                application:set_env(emqx, enable_acl_cache, false),
+                ok;
+           (_) ->
+                ok
+        end,
+    emqx_ct_helpers:start_apps([], Handler),
+    Config;
+t_connack_auth_error({'end', _Config}) ->
+    emqx_ct_helpers:stop_apps([]),
+    emqx_ct_helpers:boot_modules(all),
+    emqx_ct_helpers:start_apps([]),
+    ok;
+t_connack_auth_error(Config) when is_list(Config) ->
+    %% MQTT 3.1
+    ?assertEqual(0, emqx_metrics:val('packets.connack.auth_error')),
+    {ok, C0} = emqtt:start_link([{proto_ver, v4}]),
+    ?assertEqual({error, {unauthorized_client, undefined}}, emqtt:connect(C0)),
+    ?assertEqual(1, emqx_metrics:val('packets.connack.auth_error')),
+    %% MQTT 5.0
+    {ok, C1} = emqtt:start_link([{proto_ver, v5}]),
+    ?assertEqual({error, {not_authorized, #{}}}, emqtt:connect(C1)),
+    ?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')),
+    ok.
+
 recv_msgs(Count) ->
     recv_msgs(Count, []).
 

From 1733f19608ebcdfd7f59c409f945f3ef987bc9c5 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 10 Jun 2022 16:15:17 -0300
Subject: [PATCH 17/25] chore: bump version and update changelog

---
 CHANGES-4.3.md     | 5 +++++
 src/emqx.app.src   | 2 +-
 src/emqx.appup.src | 6 ++++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 00cb8cf1e..3178d817e 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -18,6 +18,11 @@ File format:
   password-protected private key files used for dashboard and
   management HTTPS listeners. [#8129]
 
+### Bug-fixes
+
+- Correctly tally `connack.auth_error` metrics when a client uses MQTT
+  3.1. [#8177]
+
 ## v4.3.15
 
 ### Enhancements
diff --git a/src/emqx.app.src b/src/emqx.app.src
index c57bd635d..5b5017b1a 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.16"}, % strict semver, bump manually!
+  {vsn, "4.3.17"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 844c7fb7a..49fc3abb3 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.15",
+  [{"4.3.16",[{load_module,emqx_metrics,brutal_purge,soft_purge,[]}]},
+   {"4.3.15",
     [{add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
@@ -555,7 +556,8 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.15",
+  [{"4.3.16",[{load_module,emqx_metrics,brutal_purge,soft_purge,[]}]},
+   {"4.3.15",
     [{delete_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},

From 9975f3dda8896cb1fd0edc1e8df73c2ad9984bda Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Sat, 11 Jun 2022 17:34:22 +0800
Subject: [PATCH 18/25] chore: fix when load plugins, update loaded file error

---
 src/emqx_plugins.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index c928d4d31..3177f05e7 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -387,7 +387,8 @@ plugin_loaded(_Name, false) ->
     ok;
 plugin_loaded(Name, true) ->
     case read_loaded() of
-        {ok, Names} ->
+        {ok, Names0} ->
+            Names = filter_plugins(Names0),
             case lists:member(Name, Names) of
                 false ->
                     %% write file if plugin is loaded

From 215286760ac497698c937251c9e7623e8c9ba9d0 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 12 Jun 2022 21:05:36 +0200
Subject: [PATCH 19/25] build: ignore _build sub-dir in release dir

---
 scripts/update_appup.escript | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript
index dd8dd9ca2..0ac5084ab 100755
--- a/scripts/update_appup.escript
+++ b/scripts/update_appup.escript
@@ -55,7 +55,10 @@ app_specific_actions(_) ->
     [].
 
 ignored_apps() ->
-    [gpb, emqx_dashboard, emqx_management] ++ otp_standard_apps().
+    [gpb, %% only a build tool
+     emqx_dashboard, %% generic appup file for all versions
+     emqx_management %% generic appup file for all versions
+    ] ++ otp_standard_apps().
 
 main(Args) ->
     #{prev_tag := Baseline} = Options = parse_args(Args, default_options()),
@@ -530,8 +533,10 @@ contains_contents(File, Upgrade, Downgrade) ->
 
 index_apps(ReleaseDir) ->
     log("INFO: indexing apps in ~s~n", [ReleaseDir]),
-    Apps0 = maps:from_list([index_app(filename:join(ReleaseDir, AppFile)) ||
-                               AppFile <- filelib:wildcard("**/ebin/*.app", ReleaseDir)]),
+    AppFiles0 = filelib:wildcard("**/ebin/*.app", ReleaseDir),
+    %% everything in _build sub-dir e.g. cuttlefish/_build should be ignored
+    AppFiles = lists:filter(fun(File) -> re:run(File, "_build") =:= nomatch end, AppFiles0),
+    Apps0 = maps:from_list([index_app(filename:join(ReleaseDir, AppFile)) || AppFile <- AppFiles]),
     maps:without(ignored_apps(), Apps0).
 
 index_app(AppFile) ->

From 5f75f03be88156cdd68a9b1c35622caffbf5cfd2 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 13 Jun 2022 15:15:01 +0800
Subject: [PATCH 20/25] chore(appup): update emqx.appup.src

---
 src/emqx.app.src   |  2 +-
 src/emqx.appup.src | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/emqx.app.src b/src/emqx.app.src
index c57bd635d..5b5017b1a 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.16"}, % strict semver, bump manually!
+  {vsn, "4.3.17"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 844c7fb7a..b72fe98d0 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,10 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.15",
+  [{"4.3.16",
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+   {"4.3.15",
     [{add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -555,10 +558,13 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.15",
+  [{"4.3.16",
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+   {"4.3.15",
     [{delete_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},

From 9bf14234526d4b89f769698505778baaa3f8a3a9 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 13 Jun 2022 15:29:22 +0800
Subject: [PATCH 21/25] chore: update changes-4.3.md

---
 CHANGES-4.3.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 00cb8cf1e..44aa5911a 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -18,6 +18,10 @@ File format:
   password-protected private key files used for dashboard and
   management HTTPS listeners. [#8129]
 
+### Bug fixes
+
+- Avoid repeated writing `loaded_plugins` file if the plugin enable stauts has not changed [#8179]
+
 ## v4.3.15
 
 ### Enhancements

From 1e02656035f4dd3d0642981c2be6b62bc17298e1 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 13 Jun 2022 10:53:36 +0800
Subject: [PATCH 22/25] fix(rule): better func name for emqx_rule_utils

---
 .../src/emqx_rule_actions.erl                 |  4 ++--
 apps/emqx_rule_engine/src/emqx_rule_utils.erl | 19 +++++--------------
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index 83c13a3af..ae82d56c9 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -261,7 +261,7 @@ to_qos(TargetQoS) ->
 
 get_qos(-1, _Data, Default) -> Default;
 get_qos(TargetQoS, Data, _Default) ->
-    qos(emqx_rule_utils:replace_simple_var(TargetQoS, Data)).
+    qos(emqx_rule_utils:replace_var(TargetQoS, Data)).
 
 qos(<<"0">>) ->  0;
 qos(<<"1">>) ->  1;
@@ -285,7 +285,7 @@ to_retain(TargetRetain) ->
     end.
 
 get_retain(TargetRetain, Data) ->
-    retain(emqx_rule_utils:replace_simple_var(TargetRetain, Data)).
+    retain(emqx_rule_utils:replace_var(TargetRetain, Data)).
 
 retain(true) ->        true;
 retain(false) ->       false;
diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
index 047b9d1af..137e22128 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
@@ -16,8 +16,7 @@
 
 -module(emqx_rule_utils).
 
--export([ replace_vars_in_str/2
-        , replace_simple_var/2
+-export([ replace_var/2
         ]).
 
 %% preprocess and process tempalte string with place holders
@@ -91,20 +90,12 @@ preproc_tmpl([[Str, Phld]| Tokens], Acc) ->
 preproc_tmpl([[Str]| Tokens], Acc) ->
     preproc_tmpl(Tokens, put_head(str, Str, Acc)).
 
-%% Replace a string contains vars to another string in which the placeholders are replace by the
-%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
-%% "a: 1".
-replace_vars_in_str(Tokens, Data) when is_list(Tokens) ->
-    proc_tmpl(Tokens, Data, #{return => full_binary});
-replace_vars_in_str(Val, _Data) ->
-    Val.
-
 %% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result
 %% value will be an integer 1.
-replace_simple_var(Tokens, Data) when is_list(Tokens) ->
-    [Var] = proc_tmpl(Tokens, Data, #{return => rawlist}),
-    Var;
-replace_simple_var(Val, _Data) ->
+replace_var(Tokens, Data) when is_list(Tokens) ->
+    [Val] = proc_tmpl(Tokens, Data, #{return => rawlist}),
+    Val;
+replace_var(Val, _Data) ->
     Val.
 
 put_head(_Type, <<>>, List) -> List;

From 38b7e5fa11d7f79e079ce2285894dfaab2280790 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 13 Jun 2022 15:00:03 +0200
Subject: [PATCH 23/25] chore: bump app vsn for emqx_dashboard

---
 apps/emqx_management/src/emqx_management.app.src   | 2 +-
 apps/emqx_management/src/emqx_management.appup.src | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index e203c7a56..3d9e6843b 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.13"}, % strict semver, bump manually!
+  {vsn, "4.3.14"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},
diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src
index 15aef0463..1463334b4 100644
--- a/apps/emqx_management/src/emqx_management.appup.src
+++ b/apps/emqx_management/src/emqx_management.appup.src
@@ -1,13 +1,13 @@
 %% -*- mode: erlang -*-
 {VSN,
- [ {<<"4\\.3\\.([0-9]|1[0-4])">>,
+ [ {<<"4\\.3\\.[0-9]+">>,
     [ {apply,{minirest,stop_http,['http:management']}},
       {apply,{minirest,stop_http,['https:management']}},
       {restart_application, emqx_management}
     ]},
    {<<".*">>, []}
  ],
- [ {<<"4\\.3\\.([0-9]|1[0-4])">>,
+ [ {<<"4\\.3\\.[0-9]+">>,
     [ {apply,{minirest,stop_http,['http:management']}},
       {apply,{minirest,stop_http,['https:management']}},
       {restart_application, emqx_management}

From 100aeda83bf5621647723bd57e99cd82e2d56761 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 13 Jun 2022 17:40:41 +0200
Subject: [PATCH 24/25] build: add strict-semver version bump check for apps

---
 scripts/apps-version-check.sh | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 48cb3218f..a16d6a5ca 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -8,6 +8,10 @@ bad_app_count=0
 no_comment_re='(^[^\s?%])'
 ## TODO: c source code comments re (in $app_path/c_src dirs)
 
+parse_semver() {
+    echo "$1" | tr '.|-' ' '
+}
+
 while read -r app; do
     if [ "$app" != "emqx" ]; then
         app_path="$app"
@@ -15,7 +19,7 @@ while read -r app; do
         app_path="."
     fi
     src_file="$app_path/src/$(basename "$app").app.src"
-    old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')"
+    old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')"
     now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
     if [ "$old_app_version" = "$now_app_version" ]; then
         changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \
@@ -36,6 +40,19 @@ while read -r app; do
             echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade"
             bad_app_count=$(( bad_app_count + 1))
         fi
+    else
+        # shellcheck disable=SC2207
+        old_app_version_semver=($(parse_semver "$old_app_version"))
+        # shellcheck disable=SC2207
+        now_app_version_semver=($(parse_semver "$now_app_version"))
+        if  [ "${old_app_version_semver[0]}" = "${now_app_version_semver[0]}" ] && \
+            [ "${old_app_version_semver[1]}" = "${now_app_version_semver[1]}" ] && \
+            [ "$(( "${old_app_version_semver[2]}" + 1 ))" = "${now_app_version_semver[2]}" ]; then
+            true
+        else
+            echo "$src_file: non-strict semver version bump from $old_app_version to $now_app_version"
+            bad_app_count=$(( bad_app_count + 1))
+        fi
     fi
 done < <(./scripts/find-apps.sh)
 

From b680ce6d61b62be4e7fa6939cdf210e9729f5c3c Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Wed, 15 Jun 2022 10:35:27 +0800
Subject: [PATCH 25/25] chore(helm): delete preStop command

---
 deploy/charts/emqx/templates/StatefulSet.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index ef4521ee8..d405eb9f3 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -214,13 +214,6 @@ spec:
             initialDelaySeconds: 60
             periodSeconds: 30
             failureThreshold: 10
-          lifecycle:
-            preStop:
-              exec:
-                command:
-                  - "/opt/emqx/bin/emqx_ctl"
-                  - "cluster"
-                  - "leave"
     {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}