feat: list rules support for pagination and fuzzy filtering

CHANGES-5.0.md

@@ -17,6 +17,10 @@
 * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518)
 * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530)
 * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528)
+* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472)
+  **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become
+  `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`,
+  which will carry the paging meta information.
 
 ## Enhancements
 

Makefile

@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.0.3
+export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf

@@ -10,6 +10,46 @@ emqx_rule_engine_api {
                            zh: "列出所有规则"
                           }
                   }
+   api1_enable {
+                   desc {
+                         en: "Filter enable/disable rules"
+                         zh: "根据规则是否开启条件过滤"
+                        }
+                  }
+
+   api1_from {
+                   desc {
+                         en: "Filter rules by from(topic), exact match"
+                         zh: "根据规则来源 Topic 过滤, 需要完全匹配"
+                        }
+                  }
+
+   api1_like_id {
+                   desc {
+                         en: "Filter rules by id, Substring matching"
+                         zh: "根据规则 id 过滤, 使用子串模糊匹配"
+                        }
+                  }
+
+   api1_like_from {
+                   desc {
+                         en: "Filter rules by from(topic), Substring matching"
+                         zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配"
+                        }
+                  }
+
+   api1_like_description {
+                   desc {
+                         en: "Filter rules by description, Substring matching"
+                         zh: "根据规则描述过滤, 使用子串模糊匹配"
+                        }
+                  }
+   api1_match_from {
+                   desc {
+                         en: "Filter rules by from(topic), Mqtt topic matching"
+                         zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配"
+                        }
+                  }
 
     api2 {
                    desc {

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -33,6 +33,9 @@
 %% API callbacks
 -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
 
+%% query callback
+-export([query/4]).
+
 -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
 -define(ERR_BADARGS(REASON), begin
     R0 = err_msg(REASON),
@@ -109,6 +112,15 @@ end).
     }
 ).
 
+-define(RULE_QS_SCHEMA, [
+    {<<"enable">>, atom},
+    {<<"from">>, binary},
+    {<<"like_id">>, binary},
+    {<<"like_from">>, binary},
+    {<<"match_from">>, binary},
+    {<<"like_description">>, binary}
+]).
+
 namespace() -> "rule".
 
 api_spec() ->
@@ -134,9 +146,31 @@ schema("/rules") ->
         get => #{
             tags => [<<"rules">>],
             description => ?DESC("api1"),
+            parameters => [
+                {enable,
+                    mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})},
+                {from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})},
+                {like_id,
+                    mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})},
+                {like_from,
+                    mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})},
+                {like_description,
+                    mk(binary(), #{
+                        desc => ?DESC("api1_like_description"), in => query, required => false
+                    })},
+                {match_from,
+                    mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})},
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit)
+            ],
             summary => <<"List Rules">>,
             responses => #{
-                200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})
+                200 =>
+                    [
+                        {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})},
+                        {meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
+                    ],
+                400 => error_schema('BAD_REQUEST', "Invalid Parameters")
             }
         },
         post => #{
@@ -236,9 +270,21 @@ param_path_id() ->
 '/rule_events'(get, _Params) ->
     {200, emqx_rule_events:event_info()}.
 
-'/rules'(get, _Params) ->
+'/rules'(get, #{query_string := QueryString}) ->
-    Records = emqx_rule_engine:get_rules_ordered_by_ts(),
+    case
-    {200, format_rule_resp(Records)};
+        emqx_mgmt_api:node_query(
+            node(),
+            QueryString,
+            ?RULE_TAB,
+            ?RULE_QS_SCHEMA,
+            {?MODULE, query}
+        )
+    of
+        {error, page_limit_invalid} ->
+            {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
+        Result ->
+            {200, Result}
+    end;
 '/rules'(post, #{body := Params0}) ->
     case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of
         <<>> ->
@@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
 
 format_rule_resp(Rules) when is_list(Rules) ->
     [format_rule_resp(R) || R <- Rules];
+format_rule_resp({Id, Rule}) ->
+    format_rule_resp(Rule#{id => Id});
 format_rule_resp(#{
     id := Id,
     name := Name,
@@ -503,3 +551,51 @@ filter_out_request_body(Conf) ->
         <<"node">>
     ],
     maps:without(ExtraConfs, Conf).
+
+query(Tab, {Qs, Fuzzy}, Start, Limit) ->
+    Ms = qs2ms(),
+    FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
+    emqx_mgmt_api:select_table_with_count(
+        Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
+    ).
+
+%% rule is not a record, so everything is fuzzy filter.
+qs2ms() ->
+    [{'_', [], ['$_']}].
+
+fuzzy_match_fun(Qs, Ms, Fuzzy) ->
+    MsC = ets:match_spec_compile(Ms),
+    fun(Rows) ->
+        Ls = ets:match_spec_run(Rows, MsC),
+        lists:filter(
+            fun(E) ->
+                run_qs_match(E, Qs) andalso
+                    run_fuzzy_match(E, Fuzzy)
+            end,
+            Ls
+        )
+    end.
+
+run_qs_match(_, []) ->
+    true;
+run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
+    Enable =:= Pattern andalso run_qs_match(E, Qs);
+run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
+    lists:member(Pattern, From) andalso run_qs_match(E, Qs);
+run_qs_match(E, [_ | Qs]) ->
+    run_qs_match(E, Qs).
+
+run_fuzzy_match(_, []) ->
+    true;
+run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
+    binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
+    binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
+    lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
+        run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) ->
+    lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso
+        run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E, [_ | Fuzzy]) ->
+    run_fuzzy_match(E, Fuzzy).

apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl

@@ -45,7 +45,7 @@ t_crud_rule_api(_Config) ->
     ),
 
     ?assertEqual(RuleID, maps:get(id, Rule)),
-    {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}),
+    {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
     ct:pal("RList : ~p", [Rules]),
     ?assert(length(Rules) > 0),
 
@@ -91,6 +91,81 @@ t_crud_rule_api(_Config) ->
     ),
     ok.
 
+t_list_rule_api(_Config) ->
+    AddIds =
+        lists:map(
+            fun(Seq0) ->
+                Seq = integer_to_binary(Seq0),
+                Params = #{
+                    <<"description">> => <<"A simple rule">>,
+                    <<"enable">> => true,
+                    <<"actions">> => [#{<<"function">> => <<"console">>}],
+                    <<"sql">> => <<"SELECT * from \"t/1\"">>,
+                    <<"name">> => <<"test_rule", Seq/binary>>
+                },
+                {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}),
+                Id
+            end,
+            lists:seq(1, 20)
+        ),
+
+    {200, #{data := Rules, meta := #{count := Count}}} =
+        emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
+    ?assertEqual(20, length(AddIds)),
+    ?assertEqual(20, length(Rules)),
+    ?assertEqual(20, Count),
+
+    [RuleID | _] = AddIds,
+    UpdateParams = #{
+        <<"description">> => <<"中文的描述也能搜索"/utf8>>,
+        <<"enable">> => false,
+        <<"actions">> => [#{<<"function">> => <<"console">>}],
+        <<"sql">> => <<"SELECT * from \"t/1/+\"">>,
+        <<"name">> => <<"test_rule_update1">>
+    },
+    {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
+        bindings => #{id => RuleID},
+        body => UpdateParams
+    }),
+    QueryStr1 = #{query_string => #{<<"enable">> => false}},
+    {200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1),
+    ?assertEqual(1, Count1),
+
+    QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
+    {200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
+    ?assertEqual(Result1, Result2),
+
+    QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
+    {200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
+    ?assertEqual(19, Count3),
+
+    QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
+    {200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
+    ?assertEqual(Result1, Result4),
+
+    QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
+    {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
+    ?assertEqual(Result1, Result5),
+
+    QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
+    {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
+    ?assertEqual(Result1, Result6),
+
+    %% clean up
+    lists:foreach(
+        fun(Id) ->
+            ?assertMatch(
+                {204},
+                emqx_rule_engine_api:'/rules/:id'(
+                    delete,
+                    #{bindings => #{id => Id}}
+                )
+            )
+        end,
+        AddIds
+    ),
+    ok.
+
 test_rule_params() ->
     #{
         body => #{