yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main-v4.3' into main4.3

This commit is contained in:
Shawn 2022-01-10 12:31:12 +08:00 committed by GitHub
parent 3956eaddd7 2ee683d2f9
commit 82c6eaa3aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.github/workflows/run_test_cases.yaml vendored
View File

@ -8,23 +8,6 @@ on:
pull_request:
jobs:
run_static_analysis:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
steps:
- uses: actions/checkout@v2
- name: set git credentials
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
fi
- name: xref
run: make xref
- name: dialyzer
run: make dialyzer
run_proper_test:
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04

8
apps/emqx_management/src/emqx_mgmt_api.erl
View File

@ -297,10 +297,12 @@ to_integer(I) when is_integer(I) ->
to_integer(B) when is_binary(B) ->
binary_to_integer(B).
%% @doc The input timestamp time is in seconds, which needs to be
%% converted to internal milliseconds here
to_timestamp(I) when is_integer(I) ->
I;
I * 1000;
to_timestamp(B) when is_binary(B) ->
binary_to_integer(B).
binary_to_integer(B) * 1000.
aton(B) when is_binary(B) ->
list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]).
@ -332,7 +334,7 @@ params2qs_test() ->
ExpectedQs = [{str, '=:=', <<"abc">>},
{int, '=:=', 123},
{atom, '=:=', connected},
{ts, '=:=', 156000},
{ts, '=:=', 156000000},
{range, '>=', 1, '=<', 5}
],
FuzzyQs = [{fuzzy, like, <<"user">>},

48
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -333,31 +333,24 @@ query({Qs, Fuzzy}, Start, Limit) ->
match_fun(Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
REFuzzy = lists:map(fun({K, like, S}) ->
{ok, RE} = re:compile(escape(S)),
{K, like, RE}
end, Fuzzy),
fun(Rows) ->
case ets:match_spec_run(Rows, MsC) of
[] -> [];
Ls ->
lists:filter(fun(E) ->
run_fuzzy_match(E, REFuzzy)
run_fuzzy_match(E, Fuzzy)
end, Ls)
end
end.
escape(B) when is_binary(B) ->
re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]).
run_fuzzy_match(_, []) ->
true;
run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) ->
Val = case maps:get(Key, ClientInfo, "") of
undefined -> "";
run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) ->
Val = case maps:get(Key, ClientInfo, undefined) of
undefined -> <<>>;
V -> V
end,
re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy).
binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy).
%%--------------------------------------------------------------------
%% QueryString to Match Spec
@ -442,10 +435,10 @@ params2qs_test() ->
proto_ver => 4,
connected_at => '$3'},
session => #{created_at => '$2'}},
ExpectedCondi = [{'>=','$2', 1},
{'=<','$2', 5},
{'>=','$3', 1},
{'=<','$3', 5}],
ExpectedCondi = [{'>=','$2', 1000},
{'=<','$2', 5000},
{'>=','$3', 1000},
{'=<','$3', 5000}],
{10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema),
[{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1),
?assertEqual(ExpectedMtchHead, MtchHead),
@ -453,9 +446,24 @@ params2qs_test() ->
[{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]).
escape_test() ->
Str = <<"\\n">>,
{ok, Re} = re:compile(escape(Str)),
{match, _} = re:run(<<"\\name">>, Re).
fuzzy_match_test() ->
Info = {emqx_channel_info,
#{clientinfo =>
#{ clientid => <<"abcde">>
, username => <<"abc\\name*[]()">>
}}, []
},
true = run_fuzzy_match(Info, [{clientid, like, <<"abcde">>}]),
true = run_fuzzy_match(Info, [{clientid, like, <<"bcd">>}]),
false = run_fuzzy_match(Info, [{clientid, like, <<"defh">>}]),
true = run_fuzzy_match(Info, [{username, like, <<"\\name">>}]),
true = run_fuzzy_match(Info, [{username, like, <<"*">>}]),
true = run_fuzzy_match(Info, [{username, like, <<"[]">>}]),
true = run_fuzzy_match(Info, [{username, like, <<"()">>}]),
false = run_fuzzy_match(Info, [{username, like, <<"))">>}]),
true = run_fuzzy_match(Info, [{clientid, like, <<"de">>},
{username, like, <<"[]">>}]).
-endif.

175
apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
View File

@ -1,78 +1,107 @@
%% -*- mode: erlang -*-
{VSN,
[{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{update, emqx_rule_metrics, {advanced, ["4.3.0"]}},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.1",
[{update, emqx_rule_metrics, {advanced, ["4.3.1"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{update, emqx_rule_metrics, {advanced, ["4.3.2"]}},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.3",
[{update, emqx_rule_metrics, {advanced, ["4.3.3"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.3.4",
[{update, emqx_rule_metrics, {advanced, ["4.3.4"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.3.5",
[{update, emqx_rule_metrics, {advanced, ["4.3.5"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
[
{"4.3.6",
[{update, emqx_rule_metrics, {advanced, ["4.3.6"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{update, emqx_rule_metrics, {advanced, ["4.3.0"]}},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.1",
[{update, emqx_rule_metrics, {advanced, ["4.3.1"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{update, emqx_rule_metrics, {advanced, ["4.3.2"]}},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{apply,{emqx_stats,cancel_update,[rule_registery_stats]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]},
{"4.3.3",
[{update, emqx_rule_metrics, {advanced, ["4.3.3"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.3.4",
[{update, emqx_rule_metrics, {advanced, ["4.3.4"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
[ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.5",
[{update, emqx_rule_metrics, {advanced, ["4.3.5"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
[ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
]},
{"4.3.4",
[ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
]},
{"4.3.3",
[ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.2",
[ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.1",
[ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.0",
[ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}}
, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{<<".*">>, []}
],
[
{"4.3.6",
[{update, emqx_rule_metrics, {advanced, ["4.3.6"]}},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.
[ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.5",
[ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
]},
{"4.3.4",
[ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
]},
{"4.3.3",
[ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.2",
[ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.1",
[ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{"4.3.0",
[ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}}
, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}
, {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}
, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}
, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
]},
{<<".*">>, []}
]
}.

13
apps/emqx_rule_engine/src/emqx_rule_engine.erl
View File

@ -491,11 +491,24 @@ may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) ->
may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) ->
%% prepare new actions before removing old ones
NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)),
ok = restore_action_metrics(OldActions, NewActions),
_ = ?CLUSTER_CALL(clear_actions, [OldActions]),
may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params));
may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params
Rule.
%% NOTE: if the user removed an action, but the action is not the last one in the list,
%% the `restore_action_metrics/2` will not work as expected!
restore_action_metrics([#action_instance{id = OldId} | OldActions],
[#action_instance{id = NewId} | NewActions]) ->
emqx_rule_metrics:inc_actions_taken(NewId, emqx_rule_metrics:get_actions_taken(OldId)),
emqx_rule_metrics:inc_actions_success(NewId, emqx_rule_metrics:get_actions_success(OldId)),
emqx_rule_metrics:inc_actions_error(NewId, emqx_rule_metrics:get_actions_error(OldId)),
emqx_rule_metrics:inc_actions_exception(NewId, emqx_rule_metrics:get_actions_exception(OldId)),
restore_action_metrics(OldActions, NewActions);
restore_action_metrics(_, _) ->
ok.
ignore_lib_apps(Apps) ->
LibApps = [kernel, stdlib, sasl, appmon, eldap, erts,
syntax_tools, ssl, crypto, mnesia, os_mon,

4
apps/emqx_rule_engine/src/emqx_rule_runtime.erl
View File

@ -215,6 +215,10 @@ match_conditions({}, _Data) ->
true.
%% comparing numbers against strings
compare(Op, undefined, undefined) ->
do_compare(Op, undefined, undefined);
compare(_Op, L, R) when L == undefined; R == undefined ->
false;
compare(Op, L, R) when is_number(L), is_binary(R) ->
do_compare(Op, L, number(R));
compare(Op, L, R) when is_binary(L), is_number(R) ->