diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e278c9502..f4baef697 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,6 +17,10 @@ * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) +* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472) + **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become + `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, + which will carry the paging meta information. ## Enhancements diff --git a/Makefile b/Makefile index a18cbee10..bde3e6d76 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.3 +export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf b/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf index c77c66d7c..96f999d41 100644 --- a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf +++ b/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf @@ -10,6 +10,46 @@ emqx_rule_engine_api { zh: "列出所有规则" } } + api1_enable { + desc { + en: "Filter enable/disable rules" + zh: "根据规则是否开启条件过滤" + } + } + + api1_from { + desc { + en: "Filter rules by from(topic), exact match" + zh: "根据规则来源 Topic 过滤, 需要完全匹配" + } + } + + api1_like_id { + desc { + en: "Filter rules by id, Substring matching" + zh: "根据规则 id 过滤, 使用子串模糊匹配" + } + } + + api1_like_from { + desc { + en: "Filter rules by from(topic), Substring matching" + zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配" + } + } + + api1_like_description { + desc { + en: "Filter rules by description, Substring matching" + zh: "根据规则描述过滤, 使用子串模糊匹配" + } + } + api1_match_from { + desc { + en: "Filter rules by from(topic), Mqtt topic matching" + zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" + } + } api2 { desc { diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 658781636..597ee838f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -33,6 +33,9 @@ %% API callbacks -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]). +%% query callback +-export([query/4]). + -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -109,6 +112,15 @@ end). } ). +-define(RULE_QS_SCHEMA, [ + {<<"enable">>, atom}, + {<<"from">>, binary}, + {<<"like_id">>, binary}, + {<<"like_from">>, binary}, + {<<"match_from">>, binary}, + {<<"like_description">>, binary} +]). + namespace() -> "rule". api_spec() -> @@ -134,9 +146,31 @@ schema("/rules") -> get => #{ tags => [<<"rules">>], description => ?DESC("api1"), + parameters => [ + {enable, + mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})}, + {from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})}, + {like_id, + mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})}, + {like_from, + mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})}, + {like_description, + mk(binary(), #{ + desc => ?DESC("api1_like_description"), in => query, required => false + })}, + {match_from, + mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})}, + ref(emqx_dashboard_swagger, page), + ref(emqx_dashboard_swagger, limit) + ], summary => <<"List Rules">>, responses => #{ - 200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")}) + 200 => + [ + {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})}, + {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} + ], + 400 => error_schema('BAD_REQUEST', "Invalid Parameters") } }, post => #{ @@ -236,9 +270,21 @@ param_path_id() -> '/rule_events'(get, _Params) -> {200, emqx_rule_events:event_info()}. -'/rules'(get, _Params) -> - Records = emqx_rule_engine:get_rules_ordered_by_ts(), - {200, format_rule_resp(Records)}; +'/rules'(get, #{query_string := QueryString}) -> + case + emqx_mgmt_api:node_query( + node(), + QueryString, + ?RULE_TAB, + ?RULE_QS_SCHEMA, + {?MODULE, query} + ) + of + {error, page_limit_invalid} -> + {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + Result -> + {200, Result} + end; '/rules'(post, #{body := Params0}) -> case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of <<>> -> @@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). format_rule_resp(Rules) when is_list(Rules) -> [format_rule_resp(R) || R <- Rules]; +format_rule_resp({Id, Rule}) -> + format_rule_resp(Rule#{id => Id}); format_rule_resp(#{ id := Id, name := Name, @@ -503,3 +551,51 @@ filter_out_request_body(Conf) -> <<"node">> ], maps:without(ExtraConfs, Conf). + +query(Tab, {Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(), + FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy), + emqx_mgmt_api:select_table_with_count( + Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1 + ). + +%% rule is not a record, so everything is fuzzy filter. +qs2ms() -> + [{'_', [], ['$_']}]. + +fuzzy_match_fun(Qs, Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter( + fun(E) -> + run_qs_match(E, Qs) andalso + run_fuzzy_match(E, Fuzzy) + end, + Ls + ) + end. + +run_qs_match(_, []) -> + true; +run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) -> + Enable =:= Pattern andalso run_qs_match(E, Qs); +run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) -> + lists:member(Pattern, From) andalso run_qs_match(E, Qs); +run_qs_match(E, [_ | Qs]) -> + run_qs_match(E, Qs). + +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) -> + binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) -> + binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) -> + lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) -> + lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E, [_ | Fuzzy]) -> + run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 13db82aa9..da4e299f9 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -45,7 +45,7 @@ t_crud_rule_api(_Config) -> ), ?assertEqual(RuleID, maps:get(id, Rule)), - {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}), + {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), @@ -91,6 +91,81 @@ t_crud_rule_api(_Config) -> ), ok. +t_list_rule_api(_Config) -> + AddIds = + lists:map( + fun(Seq0) -> + Seq = integer_to_binary(Seq0), + Params = #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => <<"test_rule", Seq/binary>> + }, + {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + Id + end, + lists:seq(1, 20) + ), + + {200, #{data := Rules, meta := #{count := Count}}} = + emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), + ?assertEqual(20, length(AddIds)), + ?assertEqual(20, length(Rules)), + ?assertEqual(20, Count), + + [RuleID | _] = AddIds, + UpdateParams = #{ + <<"description">> => <<"中文的描述也能搜索"/utf8>>, + <<"enable">> => false, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1/+\"">>, + <<"name">> => <<"test_rule_update1">> + }, + {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ + bindings => #{id => RuleID}, + body => UpdateParams + }), + QueryStr1 = #{query_string => #{<<"enable">> => false}}, + {200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1), + ?assertEqual(1, Count1), + + QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}}, + {200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2), + ?assertEqual(Result1, Result2), + + QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}}, + {200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3), + ?assertEqual(19, Count3), + + QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}}, + {200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4), + ?assertEqual(Result1, Result4), + + QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}}, + {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5), + ?assertEqual(Result1, Result5), + + QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}}, + {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6), + ?assertEqual(Result1, Result6), + + %% clean up + lists:foreach( + fun(Id) -> + ?assertMatch( + {204}, + emqx_rule_engine_api:'/rules/:id'( + delete, + #{bindings => #{id => Id}} + ) + ) + end, + AddIds + ), + ok. + test_rule_params() -> #{ body => #{