From 1d9ee25c920516e217d241f6444bdf15c3665275 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 4 Aug 2022 10:23:26 +0800 Subject: [PATCH 1/5] fix: support custom count function --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl | 12 ++++++++++-- apps/emqx_management/src/emqx_mgmt_api.erl | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index 331d14a92..b09f99466 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -132,7 +132,11 @@ list_clientid(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, - return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun)}). + CountFun = fun() -> + MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}], + ets:select_count(?TABLE, MatchSpec) + end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}). lookup_clientid(#{clientid := Clientid}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}). @@ -182,7 +186,11 @@ delete_clientid(#{clientid := Clientid}, _) -> list_username(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, - return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun)}). + CountFun = fun() -> + MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}], + ets:select_count(?TABLE, MatchSpec) + end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}). lookup_username(#{username := Username}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 66a5c1f9d..482dcac0e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -24,6 +24,7 @@ -export([ params2qs/2 , node_query/4 , node_query/5 + , node_query/6 , cluster_query/3 , traverse_table/5 , select_table/5 @@ -59,6 +60,11 @@ query_handle([Table]) when is_atom(Table) -> query_handle(Tables) -> qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). +count_size(Table, undefined) -> + count(Table); +count_size(_Table, CountFun) -> + CountFun(). + count(Table) when is_atom(Table) -> ets:info(Table, size); count([Table]) when is_atom(Table) -> @@ -83,9 +89,12 @@ limit(Params) -> %%-------------------------------------------------------------------- node_query(Node, Params, {Tab, QsSchema}, QueryFun) -> - node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined). + node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined, undefined). node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) -> + node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, undefined). + +node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, CountFun) -> {CodCnt, Qs} = params2qs(Params, QsSchema), Limit = limit(Params), Page = page(Params), @@ -95,7 +104,7 @@ node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) -> {_, Rows} = do_query(Node, Qs, QueryFun, Start, Limit+1), Meta = #{page => Page, limit => Limit}, NMeta = case CodCnt =:= 0 of - true -> Meta#{count => count(Tab), hasnext => length(Rows) > Limit}; + true -> Meta#{count => count_size(Tab, CountFun), hasnext => length(Rows) > Limit}; _ -> Meta#{count => -1, hasnext => length(Rows) > Limit} end, Data0 = lists:sublist(Rows, Limit), From f97820f9138c3d5e06d1fac4c7aabd5a95061fca Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 5 Aug 2022 11:07:16 +0800 Subject: [PATCH 2/5] test: add auth_mnesia count test --- .../test/emqx_auth_mnesia_SUITE.erl | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 1dd979379..508ff93b3 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -272,7 +272,8 @@ t_clientid_rest_api(_Config) -> clean_all_users(), {ok, Result1} = request_http_rest_list(["auth_clientid"]), - [] = get_http_data(Result1), + ?assertMatch(#{<<"data">> := [], <<"meta">> := #{<<"count">> := 0}}, + emqx_json:decode(Result1, [return_maps])), Params1 = #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD}, {ok, _} = request_http_rest_add(["auth_clientid"], Params1), @@ -295,8 +296,27 @@ t_clientid_rest_api(_Config) -> }, get_http_data(Result3)), {ok, Result4} = request_http_rest_list(["auth_clientid"]), + #{<<"data">> := Data4, <<"meta">> := #{<<"count">> := Count4}} + = emqx_json:decode(Result4, [return_maps]), - ?assertEqual(3, length(get_http_data(Result4))), + ?assertEqual(3, Count4), + ?assertEqual(3, length(Data4)), + + UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD} + , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} + , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD} + ], + {ok, _} = request_http_rest_add(["auth_username"], UserNameParams), + + {ok, Result41} = request_http_rest_list(["auth_clientid"]), + %% the count clientid is not affected by username count. + ?assertEqual(Result4, Result41), + + {ok, Result42} = request_http_rest_list(["auth_username"]), + #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}} + = emqx_json:decode(Result42, [return_maps]), + ?assertEqual(3, Count42), + ?assertEqual(3, length(Data42)), {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]), ?assertEqual(2, length(get_http_data(Result5))), From 43e1087e131960942d9a1144aec2890b4cc3b8fe Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 5 Aug 2022 11:34:04 +0800 Subject: [PATCH 3/5] test: add auth_mnesia count test --- CHANGES-4.3.md | 7 ++++++- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src | 2 +- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 7 +++++-- apps/emqx_management/src/emqx_management.app.src | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a9d89473e..6d1e0a4dc 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -8,7 +8,12 @@ File format: - Use weight-2 heading for releases - One list item per change topic - Change log ends with a list of github PRs + Change log ends with a list of GitHub PRs + +## v4.3.19 + +### Bug fixes +- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655) ## v4.3.18 diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 592535f46..54d1317d7 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.7"}, % strict semver, bump manually + {vsn, "4.3.8"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index a849002d6..8fa8384d4 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[5-6]">>, + [{<<"4\\.3\\.7">>, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, @@ -28,7 +30,8 @@ {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[5-6]">>, + [{"4.3.7",[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 937ec9db6..6a5ae3d07 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.15"}, % strict semver, bump manually! + {vsn, "4.3.16"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, From f2a248d83ff6d0e61b7f8d1f637bb92054bed3ee Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 09:35:26 +0800 Subject: [PATCH 4/5] chore: update CHANGES-4.3.md Co-authored-by: Thales Macedo Garitezi --- CHANGES-4.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 6d1e0a4dc..d711c589b 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -13,7 +13,7 @@ File format: ## v4.3.19 ### Bug fixes -- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655) +- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) ## v4.3.18 From 4869c94e974eb44fea44f4b6aca9eaaa96fb349f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 10:25:01 +0800 Subject: [PATCH 5/5] chore: replace / with _ in match_spec --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl | 4 ++-- apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index b09f99466..6e435ee11 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -133,7 +133,7 @@ list_clientid(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, CountFun = fun() -> - MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}], + MatchSpec = [{{?TABLE, {clientid, '_'}, '_', '_'}, [], [true]}], ets:select_count(?TABLE, MatchSpec) end, return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}). @@ -187,7 +187,7 @@ delete_clientid(#{clientid := Clientid}, _) -> list_username(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, CountFun = fun() -> - MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}], + MatchSpec = [{{?TABLE, {username, '_'}, '_', '_'}, [], [true]}], ets:select_count(?TABLE, MatchSpec) end, return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}). diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 508ff93b3..0253110aa 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -300,10 +300,10 @@ t_clientid_rest_api(_Config) -> = emqx_json:decode(Result4, [return_maps]), ?assertEqual(3, Count4), - ?assertEqual(3, length(Data4)), + ?assertEqual([<<"client2">>, <<"clientid1">>, ?CLIENTID], + lists:sort(lists:map(fun(#{<<"clientid">> := C}) -> C end, Data4))), - UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD} - , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} + UserNameParams = [#{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD} ], {ok, _} = request_http_rest_add(["auth_username"], UserNameParams), @@ -315,8 +315,9 @@ t_clientid_rest_api(_Config) -> {ok, Result42} = request_http_rest_list(["auth_username"]), #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}} = emqx_json:decode(Result42, [return_maps]), - ?assertEqual(3, Count42), - ?assertEqual(3, length(Data42)), + ?assertEqual(2, Count42), + ?assertEqual([<<"username1">>, <<"username2">>], + lists:sort(lists:map(fun(#{<<"username">> := U}) -> U end, Data42))), {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]), ?assertEqual(2, length(get_http_data(Result5))),