feat: list rules support for pagination and fuzzy filtering

CHANGES-4.3.md

@@ -19,6 +19,9 @@ File format:
 
 - Fixed crash when shared persistent subscription [#8441]
 
+### Enhancements
+- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
+
 ## v4.3.16
 
 ### Enhancements

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -2,14 +2,16 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.11",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
@@ -162,14 +164,16 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.11",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -184,6 +184,17 @@
         ]).
 
 -export([list_events/2]).
+-export([query/3]).
+
+-define(RULE_QS_SCHEMA, {?RULE_TAB,
+    [
+        {<<"enabled">>, atom},
+        {<<"for">>, binary},
+        {<<"_like_id">>, binary},
+        {<<"_like_for">>, binary},
+        {<<"_match_for">>, binary},
+        {<<"_like_description">>, binary}
+    ]}).
 
 -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))).
 -define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))).
@@ -261,8 +272,11 @@ update_rule(#{id := Id}, Params) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-list_rules(_Bindings, _Params) ->
+list_rules(_Bindings, []) ->
-    return_all(emqx_rule_registry:get_rules_ordered_by_ts()).
+    return_all(emqx_rule_registry:get_rules_ordered_by_ts());
+list_rules(_Bindings, Params) ->
+    SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}).
 
 show_rule(#{id := Id}, _Params) ->
     reply_with(fun emqx_rule_registry:get_rule/1, Id).
@@ -454,6 +468,7 @@ record_to_map(#rule{id = Id,
                     actions = Actions,
                     on_action_failed = OnFailed,
                     enabled = Enabled,
+                    created_at = CreatedAt,
                     description = Descr}) ->
     #{id => Id,
       for => Hook,
@@ -462,6 +477,7 @@ record_to_map(#rule{id = Id,
       on_action_failed => OnFailed,
       metrics => get_rule_metrics(Id),
       enabled => Enabled,
+      created_at => CreatedAt,
       description => Descr
      };
 
@@ -599,3 +615,41 @@ get_action_metrics(Id) ->
             Res -> [maps:put(node, Node, Res)]
         end
         || Node <- ekka_mnesia:running_nodes()]).
+
+query({Qs, []}, Start, Limit) ->
+    Ms = qs2ms(Qs),
+    emqx_mgmt_api:select_table(?RULE_TAB, Ms, Start, Limit, fun record_to_map/1);
+
+query({Qs, Fuzzy}, Start, Limit) ->
+    Ms = qs2ms(Qs),
+    MatchFun = match_fun(Ms, Fuzzy),
+    emqx_mgmt_api:traverse_table(?RULE_TAB, MatchFun, Start, Limit, fun record_to_map/1).
+
+qs2ms(Qs) ->
+    Init = #rule{for = '_', enabled = '_', _ = '_'},
+    MatchHead = lists:foldl(fun(Q, Acc) ->  match_ms(Q, Acc) end, Init, Qs),
+    [{MatchHead, [], ['$_']}].
+
+match_ms({for, '=:=', Value}, MatchHead) -> MatchHead#rule{for = Value};
+match_ms({enabled, '=:=', Value}, MatchHead) -> MatchHead#rule{enabled = Value};
+match_ms(_, MatchHead) -> MatchHead.
+
+match_fun(Ms, Fuzzy) ->
+    MsC = ets:match_spec_compile(Ms),
+    fun(Rows) ->
+        Ls = ets:match_spec_run(Rows, MsC),
+        lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls)
+    end.
+
+run_fuzzy_match(_, []) -> true;
+run_fuzzy_match(E = #rule{id = Id}, [{id, like, Pattern}|Fuzzy]) ->
+    binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = #rule{description = Desc}, [{description, like, Pattern}|Fuzzy]) ->
+    binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = #rule{for = Topics}, [{for, match, Pattern}|Fuzzy]) ->
+    lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics)
+        andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = #rule{for = Topics}, [{for, like, Pattern}|Fuzzy]) ->
+    lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics)
+        andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false.

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -62,7 +62,8 @@ groups() ->
        t_show_action_api,
        t_crud_resources_api,
        t_list_resource_types_api,
-       t_show_resource_type_api
+       t_show_resource_type_api,
+       t_list_rule_api
        ]},
      {cli, [],
       [t_rules_cli,
@@ -513,6 +514,75 @@ t_crud_rule_api(_Config) ->
     ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound),
     ok.
 
+t_list_rule_api(_Config) ->
+    AddIds =
+        lists:map(fun(Seq) ->
+            SeqBin = integer_to_binary(Seq),
+            {ok, #{code := 0, data := #{id := Id}}} =
+                emqx_rule_engine_api:create_rule(#{},
+                    [{<<"name">>, <<"debug-rule-", SeqBin/binary>>},
+                        {<<"rawsql">>, <<"select * from \"t/a/", SeqBin/binary, "\"">>},
+                        {<<"actions">>, [[{<<"name">>,<<"inspect">>}, {<<"params">>,[{<<"arg1">>,1}]}]]},
+                        {<<"description">>, <<"debug rule desc ", SeqBin/binary>>}]),
+            Id
+                  end, lists:seq(1, 20)),
+
+    {ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{},
+        [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}]),
+    ?assertEqual(10, length(Rules11)),
+    {ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{},
+        [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}]),
+    ?assertEqual(10, length(Rules12)),
+    Rules1 = Rules11 ++ Rules12,
+
+    [RuleID | _] = AddIds,
+    {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
+        [{<<"enabled">>, false}]),
+    Params1 = [{<<"enabled">>,<<"true">>}],
+    {ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1),
+    ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)),
+
+    Params2 = [{<<"for">>, RuleID}],
+    {ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2),
+    ?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)),
+
+    Params3 = [{<<"_like_id">>,<<"rule:">>}],
+    {ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3),
+    ?assertEqual(length(Rules1), length(Rules4)),
+
+    Params4 = [{<<"_like_for">>,<<"t/a/">>}],
+    {ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4),
+    ?assertEqual(length(Rules1), length(Rules5)),
+    {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
+        [{<<"rawsql">>, <<"select * from \"t/b/c\"">>}]),
+    {ok, #{code := 0, data := Rules6}} = emqx_rule_engine_api:list_rules(#{}, Params4),
+    ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)),
+    ?assertEqual(1, length(Rules1) - length(Rules6)),
+
+    Params5 = [{<<"_match_for">>,<<"t/+/+">>}],
+    {ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5),
+    ?assertEqual(length(Rules1), length(Rules7)),
+    {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
+        [{<<"rawsql">>, <<"select * from \"t1/b\"">>}]),
+    {ok, #{code := 0, data := Rules8}} = emqx_rule_engine_api:list_rules(#{}, Params5),
+    ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)),
+    ?assertEqual(1, length(Rules1) - length(Rules8)),
+
+    Params6 = [{<<"_like_description">>,<<"rule">>}],
+    {ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6),
+    ?assertEqual(length(Rules1), length(Rules9)),
+    {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
+        [{<<"description">>, <<"not me">>}]),
+    {ok, #{code := 0, data := Rules10}} = emqx_rule_engine_api:list_rules(#{}, Params6),
+    ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules10)),
+    ?assertEqual(1, length(Rules1) - length(Rules10)),
+
+    lists:foreach(fun(ID) ->
+        ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => ID}, []))
+                  end, AddIds),
+    ok.
+
+
 t_list_actions_api(_Config) ->
     {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []),
     %ct:pal("RList : ~p", [Actions]),