yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: list rules support for pagination and fuzzy filtering

This commit is contained in:
Zhongwen Deng 2022-07-13 16:26:46 +08:00
parent 7ad0dc7c73
commit 56417a3130
5 changed files with 221 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES-5.0.md
View File

@ -17,6 +17,10 @@
* Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518)
* Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530)
* Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528)
* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472)
**‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become
`{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`,
which will carry the paging meta information.
## Enhancements

2
Makefile
View File

@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.0.3
export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
ifeq ($(OS),Windows_NT)

40
apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf
View File

@ -10,6 +10,46 @@ emqx_rule_engine_api {
zh: "列出所有规则"
}
}
api1_enable {
desc {
en: "Filter enable/disable rules"
zh: "根据规则是否开启条件过滤"
}
}
api1_from {
desc {
en: "Filter rules by from(topic), exact match"
zh: "根据规则来源 Topic 过滤, 需要完全匹配"
}
}
api1_like_id {
desc {
en: "Filter rules by id, Substring matching"
zh: "根据规则 id 过滤, 使用子串模糊匹配"
}
}
api1_like_from {
desc {
en: "Filter rules by from(topic), Substring matching"
zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配"
}
}
api1_like_description {
desc {
en: "Filter rules by description, Substring matching"
zh: "根据规则描述过滤, 使用子串模糊匹配"
}
}
api1_match_from {
desc {
en: "Filter rules by from(topic), Mqtt topic matching"
zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配"
}
}
api2 {
desc {

104
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -33,6 +33,9 @@
%% API callbacks
-export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
%% query callback
-export([query/4]).
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
-define(ERR_BADARGS(REASON), begin
R0 = err_msg(REASON),
@ -109,6 +112,15 @@ end).
}
).
-define(RULE_QS_SCHEMA, [
{<<"enable">>, atom},
{<<"from">>, binary},
{<<"like_id">>, binary},
{<<"like_from">>, binary},
{<<"match_from">>, binary},
{<<"like_description">>, binary}
]).
namespace() -> "rule".
api_spec() ->
@ -134,9 +146,31 @@ schema("/rules") ->
get => #{
tags => [<<"rules">>],
description => ?DESC("api1"),
parameters => [
{enable,
mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})},
{from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})},
{like_id,
mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})},
{like_from,
mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})},
{like_description,
mk(binary(), #{
desc => ?DESC("api1_like_description"), in => query, required => false
})},
{match_from,
mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})},
ref(emqx_dashboard_swagger, page),
ref(emqx_dashboard_swagger, limit)
],
summary => <<"List Rules">>,
responses => #{
200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})
200 =>
[
{data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})},
{meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
],
400 => error_schema('BAD_REQUEST', "Invalid Parameters")
}
},
post => #{
@ -236,9 +270,21 @@ param_path_id() ->
'/rule_events'(get, _Params) ->
{200, emqx_rule_events:event_info()}.
'/rules'(get, _Params) ->
Records = emqx_rule_engine:get_rules_ordered_by_ts(),
{200, format_rule_resp(Records)};
'/rules'(get, #{query_string := QueryString}) ->
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?RULE_TAB,
?RULE_QS_SCHEMA,
{?MODULE, query}
)
of
{error, page_limit_invalid} ->
{400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
Result ->
{200, Result}
end;
'/rules'(post, #{body := Params0}) ->
case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of
<<>> ->
@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
format_rule_resp(Rules) when is_list(Rules) ->
[format_rule_resp(R) || R <- Rules];
format_rule_resp({Id, Rule}) ->
format_rule_resp(Rule#{id => Id});
format_rule_resp(#{
id := Id,
name := Name,
@ -503,3 +551,51 @@ filter_out_request_body(Conf) ->
<<"node">>
],
maps:without(ExtraConfs, Conf).
query(Tab, {Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(),
FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
emqx_mgmt_api:select_table_with_count(
Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
).
%% rule is not a record, so everything is fuzzy filter.
qs2ms() ->
[{'_', [], ['$_']}].
fuzzy_match_fun(Qs, Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
fun(Rows) ->
Ls = ets:match_spec_run(Rows, MsC),
lists:filter(
fun(E) ->
run_qs_match(E, Qs) andalso
run_fuzzy_match(E, Fuzzy)
end,
Ls
)
end.
run_qs_match(_, []) ->
true;
run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
Enable =:= Pattern andalso run_qs_match(E, Qs);
run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
lists:member(Pattern, From) andalso run_qs_match(E, Qs);
run_qs_match(E, [_ | Qs]) ->
run_qs_match(E, Qs).
run_fuzzy_match(_, []) ->
true;
run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) ->
lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso
run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E, [_ | Fuzzy]) ->
run_fuzzy_match(E, Fuzzy).

77
apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
View File

@ -45,7 +45,7 @@ t_crud_rule_api(_Config) ->
),
?assertEqual(RuleID, maps:get(id, Rule)),
{200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}),
{200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
ct:pal("RList : ~p", [Rules]),
?assert(length(Rules) > 0),
@ -91,6 +91,81 @@ t_crud_rule_api(_Config) ->
),
ok.
t_list_rule_api(_Config) ->
AddIds =
lists:map(
fun(Seq0) ->
Seq = integer_to_binary(Seq0),
Params = #{
<<"description">> => <<"A simple rule">>,
<<"enable">> => true,
<<"actions">> => [#{<<"function">> => <<"console">>}],
<<"sql">> => <<"SELECT * from \"t/1\"">>,
<<"name">> => <<"test_rule", Seq/binary>>
},
{201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}),
Id
end,
lists:seq(1, 20)
),
{200, #{data := Rules, meta := #{count := Count}}} =
emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
?assertEqual(20, length(AddIds)),
?assertEqual(20, length(Rules)),
?assertEqual(20, Count),
[RuleID | _] = AddIds,
UpdateParams = #{
<<"description">> => <<"中文的描述也能搜索"/utf8>>,
<<"enable">> => false,
<<"actions">> => [#{<<"function">> => <<"console">>}],
<<"sql">> => <<"SELECT * from \"t/1/+\"">>,
<<"name">> => <<"test_rule_update1">>
},
{200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
bindings => #{id => RuleID},
body => UpdateParams
}),
QueryStr1 = #{query_string => #{<<"enable">> => false}},
{200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1),
?assertEqual(1, Count1),
QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
{200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
?assertEqual(Result1, Result2),
QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
{200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
?assertEqual(19, Count3),
QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
{200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
?assertEqual(Result1, Result4),
QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
{200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
?assertEqual(Result1, Result5),
QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
{200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
?assertEqual(Result1, Result6),
%% clean up
lists:foreach(
fun(Id) ->
?assertMatch(
{204},
emqx_rule_engine_api:'/rules/:id'(
delete,
#{bindings => #{id => Id}}
)
)
end,
AddIds
),
ok.
test_rule_params() ->
#{
body => #{