Merge branch 'main-v4.3' into main4.3

.github/workflows/run_test_cases.yaml

@@ -8,23 +8,6 @@ on:
   pull_request:
 
 jobs:
-    run_static_analysis:
-        runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
-
-        steps:
-        - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
-        - name: xref
-          run: make xref
-        - name: dialyzer
-          run: make dialyzer
-
     run_proper_test:
         runs-on: ubuntu-20.04
         container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04

apps/emqx_management/src/emqx_mgmt_api.erl

@@ -297,10 +297,12 @@ to_integer(I) when is_integer(I) ->
 to_integer(B) when is_binary(B) ->
     binary_to_integer(B).
 
+%% @doc The input timestamp time is in seconds, which needs to be
+%% converted to internal milliseconds here
 to_timestamp(I) when is_integer(I) ->
-    I;
+    I * 1000;
 to_timestamp(B) when is_binary(B) ->
-    binary_to_integer(B).
+    binary_to_integer(B) * 1000.
 
 aton(B) when is_binary(B) ->
     list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]).
@@ -332,7 +334,7 @@ params2qs_test() ->
     ExpectedQs = [{str, '=:=', <<"abc">>},
                   {int, '=:=', 123},
                   {atom, '=:=', connected},
-                  {ts, '=:=', 156000},
+                  {ts, '=:=', 156000000},
                   {range, '>=', 1, '=<', 5}
                  ],
     FuzzyQs = [{fuzzy, like, <<"user">>},

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -333,31 +333,24 @@ query({Qs, Fuzzy}, Start, Limit) ->
 
 match_fun(Ms, Fuzzy) ->
     MsC = ets:match_spec_compile(Ms),
-    REFuzzy = lists:map(fun({K, like, S}) ->
-                  {ok, RE} = re:compile(escape(S)),
-                  {K, like, RE}
-              end, Fuzzy),
     fun(Rows) ->
          case ets:match_spec_run(Rows, MsC) of
              [] -> [];
              Ls ->
                  lists:filter(fun(E) ->
-                    run_fuzzy_match(E, REFuzzy)
+                    run_fuzzy_match(E, Fuzzy)
                  end, Ls)
          end
     end.
 
-escape(B) when is_binary(B) ->
-    re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]).
-
 run_fuzzy_match(_, []) ->
     true;
-run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) ->
+run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) ->
-    Val = case maps:get(Key, ClientInfo, "") of
+    Val = case maps:get(Key, ClientInfo, undefined) of
-              undefined -> "";
+              undefined -> <<>>;
               V -> V
           end,
-    re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy).
+    binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy).
 
 %%--------------------------------------------------------------------
 %% QueryString to Match Spec
@@ -442,10 +435,10 @@ params2qs_test() ->
                         proto_ver => 4,
                         connected_at => '$3'},
           session => #{created_at => '$2'}},
-    ExpectedCondi = [{'>=','$2', 1},
+    ExpectedCondi = [{'>=','$2', 1000},
-                     {'=<','$2', 5},
+                     {'=<','$2', 5000},
-                     {'>=','$3', 1},
+                     {'>=','$3', 1000},
-                     {'=<','$3', 5}],
+                     {'=<','$3', 5000}],
     {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema),
     [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1),
     ?assertEqual(ExpectedMtchHead, MtchHead),
@@ -453,9 +446,24 @@ params2qs_test() ->
 
     [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]).
 
-escape_test() ->
+fuzzy_match_test() ->
-    Str = <<"\\n">>,
+    Info = {emqx_channel_info,
-    {ok, Re} = re:compile(escape(Str)),
+            #{clientinfo =>
-    {match, _} = re:run(<<"\\name">>, Re).
+              #{ clientid => <<"abcde">>
+               , username => <<"abc\\name*[]()">>
+               }}, []
+           },
+    true = run_fuzzy_match(Info, [{clientid, like, <<"abcde">>}]),
+    true = run_fuzzy_match(Info, [{clientid, like, <<"bcd">>}]),
+    false = run_fuzzy_match(Info, [{clientid, like, <<"defh">>}]),
+
+    true = run_fuzzy_match(Info, [{username, like, <<"\\name">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"*">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"[]">>}]),
+    true = run_fuzzy_match(Info, [{username, like, <<"()">>}]),
+    false = run_fuzzy_match(Info, [{username, like, <<"))">>}]),
+
+    true = run_fuzzy_match(Info, [{clientid, like, <<"de">>},
+                                  {username, like, <<"[]">>}]).
 
 -endif.

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,78 +1,107 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [
-   {"4.3.0",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.0"]}},
-     {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.1"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.2"]}},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.3",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.3"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.4",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.4"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.5",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.5"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.6"]}},
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}}
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-   {<<".*">>,[]}],
+    ]},
-  [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.0"]}},
-     {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.1"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.2"]}},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
-   {"4.3.3",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.3"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.4",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.4"]}},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.5"]}},
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}}
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.4",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.3",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.2",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.1",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.0",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}}
+    , {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {<<".*">>, []}
+  ],
+  [
    {"4.3.6",
-    [{update, emqx_rule_metrics, {advanced, ["4.3.6"]}},
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}}
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
-   {<<".*">>,[]}]}.
+    ]},
+   {"4.3.5",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.4",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.3",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.2",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.1",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.0",
+    [ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}}
+    , {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
+    , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
+    , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {<<".*">>, []}
+  ]
+}.

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -491,11 +491,24 @@ may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) ->
 may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) ->
     %% prepare new actions before removing old ones
     NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)),
+    ok = restore_action_metrics(OldActions, NewActions),
     _ = ?CLUSTER_CALL(clear_actions, [OldActions]),
     may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params));
 may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params
     Rule.
 
+%% NOTE: if the user removed an action, but the action is not the last one in the list,
+%% the `restore_action_metrics/2` will not work as expected!
+restore_action_metrics([#action_instance{id = OldId} | OldActions],
+                       [#action_instance{id = NewId} | NewActions]) ->
+    emqx_rule_metrics:inc_actions_taken(NewId, emqx_rule_metrics:get_actions_taken(OldId)),
+    emqx_rule_metrics:inc_actions_success(NewId, emqx_rule_metrics:get_actions_success(OldId)),
+    emqx_rule_metrics:inc_actions_error(NewId, emqx_rule_metrics:get_actions_error(OldId)),
+    emqx_rule_metrics:inc_actions_exception(NewId, emqx_rule_metrics:get_actions_exception(OldId)),
+    restore_action_metrics(OldActions, NewActions);
+restore_action_metrics(_, _) ->
+    ok.
+
 ignore_lib_apps(Apps) ->
     LibApps = [kernel, stdlib, sasl, appmon, eldap, erts,
                syntax_tools, ssl, crypto, mnesia, os_mon,

apps/emqx_rule_engine/src/emqx_rule_runtime.erl

@@ -215,6 +215,10 @@ match_conditions({}, _Data) ->
     true.
 
 %% comparing numbers against strings
+compare(Op, undefined, undefined) ->
+    do_compare(Op, undefined, undefined);
+compare(_Op, L, R) when L == undefined; R == undefined ->
+    false;
 compare(Op, L, R) when is_number(L), is_binary(R) ->
     do_compare(Op, L, number(R));
 compare(Op, L, R) when is_binary(L), is_number(R) ->