diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index daabb0b52..9a40964bc 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -8,23 +8,6 @@ on: pull_request: jobs: - run_static_analysis: - runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 - - steps: - - uses: actions/checkout@v2 - - name: set git credentials - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi - - name: xref - run: make xref - - name: dialyzer - run: make dialyzer - run_proper_test: runs-on: ubuntu-20.04 container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index e068c5384..c34791b57 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -297,10 +297,12 @@ to_integer(I) when is_integer(I) -> to_integer(B) when is_binary(B) -> binary_to_integer(B). +%% @doc The input timestamp time is in seconds, which needs to be +%% converted to internal milliseconds here to_timestamp(I) when is_integer(I) -> - I; + I * 1000; to_timestamp(B) when is_binary(B) -> - binary_to_integer(B). + binary_to_integer(B) * 1000. aton(B) when is_binary(B) -> list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]). @@ -332,7 +334,7 @@ params2qs_test() -> ExpectedQs = [{str, '=:=', <<"abc">>}, {int, '=:=', 123}, {atom, '=:=', connected}, - {ts, '=:=', 156000}, + {ts, '=:=', 156000000}, {range, '>=', 1, '=<', 5} ], FuzzyQs = [{fuzzy, like, <<"user">>}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 1ddd87a3d..7d3dbddc8 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -333,31 +333,24 @@ query({Qs, Fuzzy}, Start, Limit) -> match_fun(Ms, Fuzzy) -> MsC = ets:match_spec_compile(Ms), - REFuzzy = lists:map(fun({K, like, S}) -> - {ok, RE} = re:compile(escape(S)), - {K, like, RE} - end, Fuzzy), fun(Rows) -> case ets:match_spec_run(Rows, MsC) of [] -> []; Ls -> lists:filter(fun(E) -> - run_fuzzy_match(E, REFuzzy) + run_fuzzy_match(E, Fuzzy) end, Ls) end end. -escape(B) when is_binary(B) -> - re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]). - run_fuzzy_match(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> - Val = case maps:get(Key, ClientInfo, "") of - undefined -> ""; +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) -> + Val = case maps:get(Key, ClientInfo, undefined) of + undefined -> <<>>; V -> V end, - re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). + binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy). %%-------------------------------------------------------------------- %% QueryString to Match Spec @@ -442,10 +435,10 @@ params2qs_test() -> proto_ver => 4, connected_at => '$3'}, session => #{created_at => '$2'}}, - ExpectedCondi = [{'>=','$2', 1}, - {'=<','$2', 5}, - {'>=','$3', 1}, - {'=<','$3', 5}], + ExpectedCondi = [{'>=','$2', 1000}, + {'=<','$2', 5000}, + {'>=','$3', 1000}, + {'=<','$3', 5000}], {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema), [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1), ?assertEqual(ExpectedMtchHead, MtchHead), @@ -453,9 +446,24 @@ params2qs_test() -> [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). -escape_test() -> - Str = <<"\\n">>, - {ok, Re} = re:compile(escape(Str)), - {match, _} = re:run(<<"\\name">>, Re). +fuzzy_match_test() -> + Info = {emqx_channel_info, + #{clientinfo => + #{ clientid => <<"abcde">> + , username => <<"abc\\name*[]()">> + }}, [] + }, + true = run_fuzzy_match(Info, [{clientid, like, <<"abcde">>}]), + true = run_fuzzy_match(Info, [{clientid, like, <<"bcd">>}]), + false = run_fuzzy_match(Info, [{clientid, like, <<"defh">>}]), + + true = run_fuzzy_match(Info, [{username, like, <<"\\name">>}]), + true = run_fuzzy_match(Info, [{username, like, <<"*">>}]), + true = run_fuzzy_match(Info, [{username, like, <<"[]">>}]), + true = run_fuzzy_match(Info, [{username, like, <<"()">>}]), + false = run_fuzzy_match(Info, [{username, like, <<"))">>}]), + + true = run_fuzzy_match(Info, [{clientid, like, <<"de">>}, + {username, like, <<"[]">>}]). -endif. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 3aeb2cd69..f329f4b22 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,78 +1,107 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{update, emqx_rule_metrics, {advanced, ["4.3.0"]}}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{update, emqx_rule_metrics, {advanced, ["4.3.1"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{update, emqx_rule_metrics, {advanced, ["4.3.2"]}}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{update, emqx_rule_metrics, {advanced, ["4.3.3"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{update, emqx_rule_metrics, {advanced, ["4.3.4"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{update, emqx_rule_metrics, {advanced, ["4.3.5"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [ {"4.3.6", - [{update, emqx_rule_metrics, {advanced, ["4.3.6"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"4.3.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{update, emqx_rule_metrics, {advanced, ["4.3.0"]}}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{update, emqx_rule_metrics, {advanced, ["4.3.1"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{update, emqx_rule_metrics, {advanced, ["4.3.2"]}}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{update, emqx_rule_metrics, {advanced, ["4.3.3"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{update, emqx_rule_metrics, {advanced, ["4.3.4"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, {"4.3.5", - [{update, emqx_rule_metrics, {advanced, ["4.3.5"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, + {"4.3.4", + [ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.1", + [ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.0", + [ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}} + , {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>, []} + ], + [ {"4.3.6", - [{update, emqx_rule_metrics, {advanced, ["4.3.6"]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + [ {update, emqx_rule_metrics, {advanced, ["4.3.6"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.5", + [ {update, emqx_rule_metrics, {advanced, ["4.3.5"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, + {"4.3.4", + [ {update, emqx_rule_metrics, {advanced, ["4.3.4"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [ {update, emqx_rule_metrics, {advanced, ["4.3.3"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [ {update, emqx_rule_metrics, {advanced, ["4.3.2"]}} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.1", + [ {update, emqx_rule_metrics, {advanced, ["4.3.1"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.0", + [ {update, emqx_rule_metrics, {advanced, ["4.3.0"]}} + , {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} + , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} + , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>, []} + ] +}. \ No newline at end of file diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 6c8b95064..0dabb1c52 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -491,11 +491,24 @@ may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) -> may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) -> %% prepare new actions before removing old ones NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)), + ok = restore_action_metrics(OldActions, NewActions), _ = ?CLUSTER_CALL(clear_actions, [OldActions]), may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params)); may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params Rule. +%% NOTE: if the user removed an action, but the action is not the last one in the list, +%% the `restore_action_metrics/2` will not work as expected! +restore_action_metrics([#action_instance{id = OldId} | OldActions], + [#action_instance{id = NewId} | NewActions]) -> + emqx_rule_metrics:inc_actions_taken(NewId, emqx_rule_metrics:get_actions_taken(OldId)), + emqx_rule_metrics:inc_actions_success(NewId, emqx_rule_metrics:get_actions_success(OldId)), + emqx_rule_metrics:inc_actions_error(NewId, emqx_rule_metrics:get_actions_error(OldId)), + emqx_rule_metrics:inc_actions_exception(NewId, emqx_rule_metrics:get_actions_exception(OldId)), + restore_action_metrics(OldActions, NewActions); +restore_action_metrics(_, _) -> + ok. + ignore_lib_apps(Apps) -> LibApps = [kernel, stdlib, sasl, appmon, eldap, erts, syntax_tools, ssl, crypto, mnesia, os_mon, diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 6c24c899e..6c2482075 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -215,6 +215,10 @@ match_conditions({}, _Data) -> true. %% comparing numbers against strings +compare(Op, undefined, undefined) -> + do_compare(Op, undefined, undefined); +compare(_Op, L, R) when L == undefined; R == undefined -> + false; compare(Op, L, R) when is_number(L), is_binary(R) -> do_compare(Op, L, number(R)); compare(Op, L, R) when is_binary(L), is_number(R) ->