From d1254faf6f1316eff3c592e006530e5288465e23 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 8 Jul 2022 15:38:28 +0800 Subject: [PATCH] feat: list rules support for pagination and fuzzy filtering --- CHANGES-4.3.md | 3 + .../src/emqx_rule_engine.appup.src | 12 ++-- .../src/emqx_rule_engine_api.erl | 58 ++++++++++++++- .../test/emqx_rule_engine_SUITE.erl | 72 ++++++++++++++++++- 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a4cac8965..f23e5aeaa 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -19,6 +19,9 @@ File format: - Fixed crash when shared persistent subscription [#8441] +### Enhancements +- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] + ## v4.3.16 ### Enhancements diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index be070305b..fd764bae2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,14 +2,16 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -162,14 +164,16 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 6587f9c5f..fb6b6f3ca 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -184,6 +184,17 @@ ]). -export([list_events/2]). +-export([query/3]). + +-define(RULE_QS_SCHEMA, {?RULE_TAB, + [ + {<<"enabled">>, atom}, + {<<"for">>, binary}, + {<<"_like_id">>, binary}, + {<<"_like_for">>, binary}, + {<<"_match_for">>, binary}, + {<<"_like_description">>, binary} + ]}). -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))). -define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))). @@ -261,8 +272,11 @@ update_rule(#{id := Id}, Params) -> return({error, 400, ?ERR_BADARGS(Reason)}) end. -list_rules(_Bindings, _Params) -> - return_all(emqx_rule_registry:get_rules_ordered_by_ts()). +list_rules(_Bindings, []) -> + return_all(emqx_rule_registry:get_rules_ordered_by_ts()); +list_rules(_Bindings, Params) -> + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}). show_rule(#{id := Id}, _Params) -> reply_with(fun emqx_rule_registry:get_rule/1, Id). @@ -454,6 +468,7 @@ record_to_map(#rule{id = Id, actions = Actions, on_action_failed = OnFailed, enabled = Enabled, + created_at = CreatedAt, description = Descr}) -> #{id => Id, for => Hook, @@ -462,6 +477,7 @@ record_to_map(#rule{id = Id, on_action_failed => OnFailed, metrics => get_rule_metrics(Id), enabled => Enabled, + created_at => CreatedAt, description => Descr }; @@ -599,3 +615,41 @@ get_action_metrics(Id) -> Res -> [maps:put(node, Node, Res)] end || Node <- ekka_mnesia:running_nodes()]). + +query({Qs, []}, Start, Limit) -> + Ms = qs2ms(Qs), + emqx_mgmt_api:select_table(?RULE_TAB, Ms, Start, Limit, fun record_to_map/1); + +query({Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(?RULE_TAB, MatchFun, Start, Limit, fun record_to_map/1). + +qs2ms(Qs) -> + Init = #rule{for = '_', enabled = '_', _ = '_'}, + MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs), + [{MatchHead, [], ['$_']}]. + +match_ms({for, '=:=', Value}, MatchHead) -> MatchHead#rule{for = Value}; +match_ms({enabled, '=:=', Value}, MatchHead) -> MatchHead#rule{enabled = Value}; +match_ms(_, MatchHead) -> MatchHead. + +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls) + end. + +run_fuzzy_match(_, []) -> true; +run_fuzzy_match(E = #rule{id = Id}, [{id, like, Pattern}|Fuzzy]) -> + binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{description = Desc}, [{description, like, Pattern}|Fuzzy]) -> + binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, match, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, like, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 663eddd5a..7368b5911 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -62,7 +62,8 @@ groups() -> t_show_action_api, t_crud_resources_api, t_list_resource_types_api, - t_show_resource_type_api + t_show_resource_type_api, + t_list_rule_api ]}, {cli, [], [t_rules_cli, @@ -513,6 +514,75 @@ t_crud_rule_api(_Config) -> ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound), ok. +t_list_rule_api(_Config) -> + AddIds = + lists:map(fun(Seq) -> + SeqBin = integer_to_binary(Seq), + {ok, #{code := 0, data := #{id := Id}}} = + emqx_rule_engine_api:create_rule(#{}, + [{<<"name">>, <<"debug-rule-", SeqBin/binary>>}, + {<<"rawsql">>, <<"select * from \"t/a/", SeqBin/binary, "\"">>}, + {<<"actions">>, [[{<<"name">>,<<"inspect">>}, {<<"params">>,[{<<"arg1">>,1}]}]]}, + {<<"description">>, <<"debug rule desc ", SeqBin/binary>>}]), + Id + end, lists:seq(1, 20)), + + {ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}]), + ?assertEqual(10, length(Rules11)), + {ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}]), + ?assertEqual(10, length(Rules12)), + Rules1 = Rules11 ++ Rules12, + + [RuleID | _] = AddIds, + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"enabled">>, false}]), + Params1 = [{<<"enabled">>,<<"true">>}], + {ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)), + + Params2 = [{<<"for">>, RuleID}], + {ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2), + ?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)), + + Params3 = [{<<"_like_id">>,<<"rule:">>}], + {ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3), + ?assertEqual(length(Rules1), length(Rules4)), + + Params4 = [{<<"_like_for">>,<<"t/a/">>}], + {ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assertEqual(length(Rules1), length(Rules5)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t/b/c\"">>}]), + {ok, #{code := 0, data := Rules6}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)), + ?assertEqual(1, length(Rules1) - length(Rules6)), + + Params5 = [{<<"_match_for">>,<<"t/+/+">>}], + {ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assertEqual(length(Rules1), length(Rules7)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t1/b\"">>}]), + {ok, #{code := 0, data := Rules8}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)), + ?assertEqual(1, length(Rules1) - length(Rules8)), + + Params6 = [{<<"_like_description">>,<<"rule">>}], + {ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assertEqual(length(Rules1), length(Rules9)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"description">>, <<"not me">>}]), + {ok, #{code := 0, data := Rules10}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules10)), + ?assertEqual(1, length(Rules1) - length(Rules10)), + + lists:foreach(fun(ID) -> + ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => ID}, [])) + end, AddIds), + ok. + + t_list_actions_api(_Config) -> {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []), %ct:pal("RList : ~p", [Actions]),