From c01aa3c5803148143db5d5f1a018122ab2838f61 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 3 Mar 2022 14:31:21 +0800 Subject: [PATCH 1/6] style(authz_mnesia): api spec style, rm duplicated module name --- apps/emqx_authz/include/emqx_authz.hrl | 21 ++ apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 326 +++++++++--------- apps/emqx_authz/src/emqx_authz_mnesia.erl | 9 +- 3 files changed, 198 insertions(+), 158 deletions(-) diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 89fb97f82..7b584b59b 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -define(APP, emqx_authz). -define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= <<"allow">>) orelse @@ -8,6 +24,10 @@ (A =:= all) orelse (A =:= <<"all">>) )). +%% authz_mnesia +-define(ACL_TABLE, emqx_acl). + +%% authz_cmd -define(CMD_REPLACE, replace). -define(CMD_DELETE, delete). -define(CMD_PREPEND, prepend). @@ -23,6 +43,7 @@ -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9_]+\\}"). +%% API examples -define(USERNAME_RULES_EXAMPLE, #{username => user1, rules => [ #{topic => <<"test/toopic/1">>, permission => <<"allow">>, diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index cd70a5340..430ebe76e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -22,6 +22,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("typerefl/include/types.hrl"). +-import(hoconsc, [mk/1, mk/2, ref/1, ref/2, array/1, enum/1]). + -define(FORMAT_USERNAME_FUN, {?MODULE, format_by_username}). -define(FORMAT_CLIENTID_FUN, {?MODULE, format_by_clientid}). @@ -68,178 +70,191 @@ paths() -> %%-------------------------------------------------------------------- schema("/authorization/sources/built-in-database/username") -> - #{ - 'operationId' => users, - get => #{ - tags => [<<"authorization">>], - description => <<"Show the list of record for username">>, - parameters => [ hoconsc:ref(emqx_dashboard_swagger, page) - , hoconsc:ref(emqx_dashboard_swagger, limit)], - responses => #{ - 200 => swagger_with_example( {username_response_data, ?TYPE_REF} - , {username, ?PAGE_QUERY_EXAMPLE}) + #{ 'operationId' => users + , get => + #{ tags => [<<"authorization">>] + , description => <<"Show the list of record for username">> + , parameters => + [ ref(emqx_dashboard_swagger, page) + , ref(emqx_dashboard_swagger, limit) + , { like_username + , mk( binary(), #{ in => query + , required => false + , desc => <<"Fuzzy search `username` as substring">>})} + ] + , responses => + #{ 200 => swagger_with_example( {username_response_data, ?TYPE_REF} + , {username, ?PAGE_QUERY_EXAMPLE}) } - }, - post => #{ - tags => [<<"authorization">>], - description => <<"Add new records for username">>, - 'requestBody' => swagger_with_example( {rules_for_username, ?TYPE_ARRAY} - , {username, ?POST_ARRAY_EXAMPLE}), - responses => #{ - 204 => <<"Created">>, - 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST] - , <<"Bad username or bad rule schema">>) + } + , post => + #{ tags => [<<"authorization">>] + , description => <<"Add new records for username">> + , 'requestBody' => swagger_with_example( {rules_for_username, ?TYPE_ARRAY} + , {username, ?POST_ARRAY_EXAMPLE}) + , responses => + #{ 204 => <<"Created">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad username or bad rule schema">>) } } }; schema("/authorization/sources/built-in-database/clientid") -> - #{ - 'operationId' => clients, - get => #{ - tags => [<<"authorization">>], - description => <<"Show the list of record for clientid">>, - parameters => [ hoconsc:ref(emqx_dashboard_swagger, page) - , hoconsc:ref(emqx_dashboard_swagger, limit)], - responses => #{ - 200 => swagger_with_example( {clientid_response_data, ?TYPE_REF} - , {clientid, ?PAGE_QUERY_EXAMPLE}) + #{ 'operationId' => clients + , get => + #{ tags => [<<"authorization">>] + , description => <<"Show the list of record for clientid">> + , parameters => + [ ref(emqx_dashboard_swagger, page) + , ref(emqx_dashboard_swagger, limit) + , { like_clientid + , mk( binary() + , #{ in => query + , required => false + , desc => <<"Fuzzy search `clientid` as substring">>}) + } + ] + , responses => + #{ 200 => swagger_with_example( {clientid_response_data, ?TYPE_REF} + , {clientid, ?PAGE_QUERY_EXAMPLE}) + } } - }, - post => #{ - tags => [<<"authorization">>], - description => <<"Add new records for clientid">>, - 'requestBody' => swagger_with_example( {rules_for_clientid, ?TYPE_ARRAY} - , {clientid, ?POST_ARRAY_EXAMPLE}), - responses => #{ - 204 => <<"Created">>, - 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST] - , <<"Bad clientid or bad rule schema">>) + , post => + #{ tags => [<<"authorization">>] + , description => <<"Add new records for clientid">> + , 'requestBody' => swagger_with_example( {rules_for_clientid, ?TYPE_ARRAY} + , {clientid, ?POST_ARRAY_EXAMPLE}) + , responses => + #{ 204 => <<"Created">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad clientid or bad rule schema">>) + } } - } - }; + }; schema("/authorization/sources/built-in-database/username/:username") -> - #{ - 'operationId' => user, - get => #{ - tags => [<<"authorization">>], - description => <<"Get record info for username">>, - parameters => [hoconsc:ref(username)], - responses => #{ - 200 => swagger_with_example( {rules_for_username, ?TYPE_REF} - , {username, ?PUT_MAP_EXAMPLE}), - 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>) + #{ 'operationId' => user + , get => + #{ tags => [<<"authorization">>] + , description => <<"Get record info for username">> + , parameters => [ref(username)] + , responses => + #{ 200 => swagger_with_example( {rules_for_username, ?TYPE_REF} + , {username, ?PUT_MAP_EXAMPLE}) + , 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Not Found">>) + } } - }, - put => #{ - tags => [<<"authorization">>], - description => <<"Set record for username">>, - parameters => [hoconsc:ref(username)], - 'requestBody' => swagger_with_example( {rules_for_username, ?TYPE_REF} - , {username, ?PUT_MAP_EXAMPLE}), - responses => #{ - 204 => <<"Updated">>, - 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST] - , <<"Bad username or bad rule schema">>) + , put => + #{ tags => [<<"authorization">>] + , description => <<"Set record for username">> + , parameters => [ref(username)] + , 'requestBody' => swagger_with_example( {rules_for_username, ?TYPE_REF} + , {username, ?PUT_MAP_EXAMPLE}) + , responses => + #{ 204 => <<"Updated">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad username or bad rule schema">>) + } } - }, - delete => #{ - tags => [<<"authorization">>], - description => <<"Delete one record for username">>, - parameters => [hoconsc:ref(username)], - responses => #{ - 204 => <<"Deleted">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad username">>) + , delete => + #{ tags => [<<"authorization">>] + , description => <<"Delete one record for username">> + , parameters => [ref(username)] + , responses => + #{ 204 => <<"Deleted">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad username">>) + } } - } - }; + }; schema("/authorization/sources/built-in-database/clientid/:clientid") -> - #{ - 'operationId' => client, - get => #{ - tags => [<<"authorization">>], - description => <<"Get record info for clientid">>, - parameters => [hoconsc:ref(clientid)], - responses => #{ - 200 => swagger_with_example( {rules_for_clientid, ?TYPE_REF} - , {clientid, ?PUT_MAP_EXAMPLE}), - 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>) + #{ 'operationId' => client + , get => + #{ tags => [<<"authorization">>] + , description => <<"Get record info for clientid">> + , parameters => [ref(clientid)] + , responses => + #{ 200 => swagger_with_example( {rules_for_clientid, ?TYPE_REF} + , {clientid, ?PUT_MAP_EXAMPLE}) + , 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Not Found">>) + } + }, + put => + #{ tags => [<<"authorization">>] + , description => <<"Set record for clientid">> + , parameters => [ref(clientid)] + , 'requestBody' => swagger_with_example( {rules_for_clientid, ?TYPE_REF} + , {clientid, ?PUT_MAP_EXAMPLE}) + , responses => + #{ 204 => <<"Updated">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad clientid or bad rule schema">>) + } } - }, - put => #{ - tags => [<<"authorization">>], - description => <<"Set record for clientid">>, - parameters => [hoconsc:ref(clientid)], - 'requestBody' => swagger_with_example( {rules_for_clientid, ?TYPE_REF} - , {clientid, ?PUT_MAP_EXAMPLE}), - responses => #{ - 204 => <<"Updated">>, - 400 => emqx_dashboard_swagger:error_codes( - [?BAD_REQUEST], <<"Bad clientid or bad rule schema">>) + , delete => + #{ tags => [<<"authorization">>] + , description => <<"Delete one record for clientid">> + , parameters => [ref(clientid)] + , responses => + #{ 204 => <<"Deleted">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad clientid">>) + } } - }, - delete => #{ - tags => [<<"authorization">>], - description => <<"Delete one record for clientid">>, - parameters => [hoconsc:ref(clientid)], - responses => #{ - 204 => <<"Deleted">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad clientid">>) - } - } - }; + }; schema("/authorization/sources/built-in-database/all") -> - #{ - 'operationId' => all, - get => #{ - tags => [<<"authorization">>], - description => <<"Show the list of rules for all">>, - responses => #{ - 200 => swagger_with_example({rules_for_all, ?TYPE_REF}, {all, ?PUT_MAP_EXAMPLE}) + #{ 'operationId' => all + , get => + #{ tags => [<<"authorization">>] + , description => <<"Show the list of rules for all">> + , responses => + #{200 => swagger_with_example({rules, ?TYPE_REF}, {all, ?PUT_MAP_EXAMPLE})} } - }, - put => #{ - tags => [<<"authorization">>], - description => <<"Set the list of rules for all">>, - 'requestBody' => - swagger_with_example({rules_for_all, ?TYPE_REF}, {all, ?PUT_MAP_EXAMPLE}), - responses => #{ - 204 => <<"Created">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad rule schema">>) + , put => + #{ tags => [<<"authorization">>] + , description => <<"Set the list of rules for all">> + , 'requestBody' => + swagger_with_example({rules, ?TYPE_REF}, {all, ?PUT_MAP_EXAMPLE}) + , responses => + #{ 204 => <<"Created">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad rule schema">>) + } } - } - }; + }; schema("/authorization/sources/built-in-database/purge-all") -> - #{ - 'operationId' => purge, - delete => #{ - tags => [<<"authorization">>], - description => <<"Purge all records">>, - responses => #{ - 204 => <<"Deleted">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + #{ 'operationId' => purge + , delete => + #{ tags => [<<"authorization">>] + , description => <<"Purge all records">> + , responses => + #{ 204 => <<"Deleted">> + , 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad Request">>) + } } - } - }. + }. fields(rule_item) -> - [ {topic, hoconsc:mk(string(), + [ {topic, mk(string(), #{ required => true , desc => <<"Rule on specific topic">> , example => <<"test/topic/1">> })} - , {permission, hoconsc:mk(hoconsc:enum([allow, deny]), + , {permission, mk(enum([allow, deny]), #{ desc => <<"Permission">> , required => true , example => allow })} - , {action, hoconsc:mk(hoconsc:enum([publish, subscribe, all]), + , {action, mk(enum([publish, subscribe, all]), #{ required => true , example => publish , desc => <<"Authorized action">> })} ]; fields(clientid) -> - [ {clientid, hoconsc:mk(binary(), + [ {clientid, mk(binary(), #{ in => path , required => true , desc => <<"ClientID">> @@ -247,50 +262,49 @@ fields(clientid) -> })} ]; fields(username) -> - [ {username, hoconsc:mk(binary(), + [ {username, mk(binary(), #{ in => path , required => true , desc => <<"Username">> , example => <<"user1">>})} ]; fields(rules_for_username) -> - [ {rules, hoconsc:mk(hoconsc:array(hoconsc:ref(rule_item)), #{})} - ] ++ fields(username); + fields(rules) + ++ fields(username); fields(username_response_data) -> - [ {data, hoconsc:mk(hoconsc:array(hoconsc:ref(rules_for_username)), #{})} - , {meta, hoconsc:ref(meta)} + [ {data, mk(array(ref(rules_for_username)), #{})} + , {meta, ref(meta)} ]; fields(rules_for_clientid) -> - [ {rules, hoconsc:mk(hoconsc:array(hoconsc:ref(rule_item)), #{})} - ] ++ fields(clientid); + fields(rules) + ++ fields(clientid); fields(clientid_response_data) -> - [ {data, hoconsc:mk(hoconsc:array(hoconsc:ref(rules_for_clientid)), #{})} - , {meta, hoconsc:ref(meta)} - ]; -fields(rules_for_all) -> - [ {rules, hoconsc:mk(hoconsc:array(hoconsc:ref(rule_item)), #{})} + [ {data, mk(array(ref(rules_for_clientid)), #{})} + , {meta, ref(meta)} ]; +fields(rules) -> + [{rules, mk(array(ref(rule_item)))}]; fields(meta) -> emqx_dashboard_swagger:fields(page) ++ emqx_dashboard_swagger:fields(limit) - ++ [{count, hoconsc:mk(integer(), #{example => 1})}]. + ++ [{count, mk(integer(), #{example => 1})}]. %%-------------------------------------------------------------------- %% HTTP API %%-------------------------------------------------------------------- -users(get, #{query_string := PageParams}) -> +users(get, #{query_string := QString}) -> {Table, MatchSpec} = emqx_authz_mnesia:list_username_rules(), - {200, emqx_mgmt_api:paginate(Table, MatchSpec, PageParams, ?FORMAT_USERNAME_FUN)}; + {200, emqx_mgmt_api:paginate(Table, MatchSpec, QString, ?FORMAT_USERNAME_FUN)}; users(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)) end, Body), {204}. -clients(get, #{query_string := PageParams}) -> +clients(get, #{query_string := QueryString}) -> {Table, MatchSpec} = emqx_authz_mnesia:list_clientid_rules(), - {200, emqx_mgmt_api:paginate(Table, MatchSpec, PageParams, ?FORMAT_CLIENTID_FUN)}; + {200, emqx_mgmt_api:paginate(Table, MatchSpec, QueryString, ?FORMAT_CLIENTID_FUN)}; clients(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"clientid">> := Clientid, <<"rules">> := Rules}) -> emqx_authz_mnesia:store_rules({clientid, Clientid}, format_rules(Rules)) @@ -402,8 +416,8 @@ atom(A) when is_atom(A) -> A. swagger_with_example({Ref, TypeP}, {_Name, _Type} = Example) -> emqx_dashboard_swagger:schema_with_examples( case TypeP of - ?TYPE_REF -> hoconsc:ref(?MODULE, Ref); - ?TYPE_ARRAY -> hoconsc:array(hoconsc:ref(?MODULE, Ref)) + ?TYPE_REF -> ref(?MODULE, Ref); + ?TYPE_ARRAY -> array(ref(?MODULE, Ref)) end, rules_example(Example)). diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index 470dcd871..bcb329202 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -20,9 +20,9 @@ -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("emqx/include/logger.hrl"). --define(ACL_SHARDED, emqx_acl_sharded). +-include("emqx_authz.hrl"). --define(ACL_TABLE, emqx_acl). +-define(ACL_SHARDED, emqx_acl_sharded). %% To save some space, use an integer for label, 0 for 'all', {1, Username} and {2, ClientId}. -define(ACL_TABLE_ALL, 0). @@ -114,10 +114,12 @@ authorize(#{username := Username, %% Management API %%-------------------------------------------------------------------- +%% Init -spec(init_tables() -> ok). init_tables() -> ok = mria_rlog:wait_for_shards([?ACL_SHARDED], infinity). +%% @doc Update authz rules -spec(store_rules(who(), rules()) -> ok). store_rules({username, Username}, Rules) -> Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = normalize_rules(Rules)}, @@ -129,6 +131,7 @@ store_rules(all, Rules) -> Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = normalize_rules(Rules)}, mria:dirty_write(Record). +%% @doc Clean all authz rules for (username & clientid & all) -spec(purge_rules() -> ok). purge_rules() -> ok = lists:foreach( @@ -137,6 +140,7 @@ purge_rules() -> end, mnesia:dirty_all_keys(?ACL_TABLE)). +%% @doc Get one record -spec(get_rules(who()) -> {ok, rules()} | not_found). get_rules({username, Username}) -> do_get_rules({?ACL_TABLE_USERNAME, Username}); @@ -145,6 +149,7 @@ get_rules({clientid, Clientid}) -> get_rules(all) -> do_get_rules(?ACL_TABLE_ALL). +%% @doc Delete one record -spec(delete_rules(who()) -> ok). delete_rules({username, Username}) -> mria:dirty_delete(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}); From 16ec8fe28983741f107af2d8ec13afa240e83840 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 4 Mar 2022 15:43:09 +0800 Subject: [PATCH 2/6] feat(authz): authz mnesia rules searching by `clientid` or `username` --- apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 82 ++++++++++++++++--- apps/emqx_authz/src/emqx_authz_mnesia.erl | 22 +++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index 430ebe76e..1f33c7a1f 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -24,8 +24,12 @@ -import(hoconsc, [mk/1, mk/2, ref/1, ref/2, array/1, enum/1]). --define(FORMAT_USERNAME_FUN, {?MODULE, format_by_username}). --define(FORMAT_CLIENTID_FUN, {?MODULE, format_by_clientid}). +-define(QUERY_USERNAME_FUN, {?MODULE, query_username}). +-define(QUERY_CLIENTID_FUN, {?MODULE, query_clientid}). + +-define(ACL_USERNAME_QSCHEMA, [{<<"like_username">>, binary}]). +-define(ACL_CLIENTID_QSCHEMA, [{<<"like_clientid">>, binary}]). + -export([ api_spec/0 , paths/0 @@ -42,8 +46,11 @@ , purge/2 ]). --export([ format_by_username/1 - , format_by_clientid/1]). +%% query funs +-export([ query_username/4 + , query_clientid/4]). + +-export([format_result/1]). -define(BAD_REQUEST, 'BAD_REQUEST'). -define(NOT_FOUND, 'NOT_FOUND'). @@ -293,9 +300,10 @@ fields(meta) -> %% HTTP API %%-------------------------------------------------------------------- -users(get, #{query_string := QString}) -> - {Table, MatchSpec} = emqx_authz_mnesia:list_username_rules(), - {200, emqx_mgmt_api:paginate(Table, MatchSpec, QString, ?FORMAT_USERNAME_FUN)}; +users(get, #{query_string := QueryString}) -> + Response = emqx_mgmt_api:node_query(node(), QueryString, + ?ACL_TABLE, ?ACL_USERNAME_QSCHEMA, ?QUERY_USERNAME_FUN), + emqx_mgmt_util:generate_response(Response); users(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)) @@ -303,8 +311,9 @@ users(post, #{body := Body}) when is_list(Body) -> {204}. clients(get, #{query_string := QueryString}) -> - {Table, MatchSpec} = emqx_authz_mnesia:list_clientid_rules(), - {200, emqx_mgmt_api:paginate(Table, MatchSpec, QueryString, ?FORMAT_CLIENTID_FUN)}; + Response = emqx_mgmt_api:node_query(node(), QueryString, + ?ACL_TABLE, ?ACL_CLIENTID_QSCHEMA, ?QUERY_CLIENTID_FUN), + emqx_mgmt_util:generate_response(Response); clients(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"clientid">> := Clientid, <<"rules">> := Rules}) -> emqx_authz_mnesia:store_rules({clientid, Clientid}, format_rules(Rules)) @@ -379,6 +388,54 @@ purge(delete, _) -> }} end. +%%-------------------------------------------------------------------- +%% Query Functions + +query_username(Tab, {_QString, []}, Continuation, Limit) -> + Ms = emqx_authz_mnesia:list_username_rules(), + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, + fun format_result/1); + +query_username(Tab, {_QString, FuzzyQString}, Continuation, Limit) -> + Ms = emqx_authz_mnesia:list_username_rules(), + FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, + fun format_result/1). + +query_clientid(Tab, {_QString, []}, Continuation, Limit) -> + Ms = emqx_authz_mnesia:list_clientid_rules(), + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, + fun format_result/1); + +query_clientid(Tab, {_QString, FuzzyQString}, Continuation, Limit) -> + Ms = emqx_authz_mnesia:list_clientid_rules(), + FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, + fun format_result/1). + +%%-------------------------------------------------------------------- +%% Match funcs + +%% Fuzzy username funcs +fuzzy_filter_fun(Fuzzy) -> + fun(MsRaws) when is_list(MsRaws) -> + lists:filter( fun(E) -> run_fuzzy_filter(E, Fuzzy) end + , MsRaws) + end. + +run_fuzzy_filter(_, []) -> + true; +run_fuzzy_filter( E = [{username, Username}, _Rule] + , [{username, like, UsernameSubStr} | Fuzzy]) -> + binary:match(Username, UsernameSubStr) /= nomatch andalso run_fuzzy_filter(E, Fuzzy); +run_fuzzy_filter( E = [{clientid, ClientId}, _Rule] + , [{clientid, like, ClientIdSubStr} | Fuzzy]) -> + binary:match(ClientId, ClientIdSubStr) /= nomatch andalso run_fuzzy_filter(E, Fuzzy). + +%%-------------------------------------------------------------------- +%% format funcs + +%% format rule from api format_rules(Rules) when is_list(Rules) -> lists:foldl(fun(#{<<"topic">> := Topic, <<"action">> := Action, @@ -388,14 +445,15 @@ format_rules(Rules) when is_list(Rules) -> AccIn ++ [{ atom(Permission), atom(Action), Topic }] end, [], Rules). -format_by_username([{username, Username}, {rules, Rules}]) -> +%% format result from mnesia tab +format_result([{username, Username}, {rules, Rules}]) -> #{username => Username, rules => [ #{topic => Topic, action => Action, permission => Permission } || {Permission, Action, Topic} <- Rules] - }. -format_by_clientid([{clientid, Clientid}, {rules, Rules}]) -> + }; +format_result([{clientid, Clientid}, {rules, Rules}]) -> #{clientid => Clientid, rules => [ #{topic => Topic, action => Action, diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index bcb329202..72c33c4b8 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -158,21 +158,19 @@ delete_rules({clientid, Clientid}) -> delete_rules(all) -> mria:dirty_delete(?ACL_TABLE, ?ACL_TABLE_ALL). --spec(list_username_rules() -> {mria:table(), ets:match_spec()}). +-spec(list_username_rules() -> ets:match_spec()). list_username_rules() -> - MatchSpec = ets:fun2ms( - fun(#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}) -> - [{username, Username}, {rules, Rules}] - end), - {?ACL_TABLE, MatchSpec}. + ets:fun2ms( + fun(#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}) -> + [{username, Username}, {rules, Rules}] + end). --spec(list_clientid_rules() -> {mria:table(), ets:match_spec()}). +-spec(list_clientid_rules() -> ets:match_spec()). list_clientid_rules() -> - MatchSpec = ets:fun2ms( - fun(#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}) -> - [{clientid, Clientid}, {rules, Rules}] - end), - {?ACL_TABLE, MatchSpec}. + ets:fun2ms( + fun(#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}) -> + [{clientid, Clientid}, {rules, Rules}] + end). -spec(record_count() -> non_neg_integer()). record_count() -> From 593e1a3efbbf4297ca4600677ff4076e39a45827 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 7 Mar 2022 11:27:10 +0800 Subject: [PATCH 3/6] feat(authn): authn mnesia rows fuzzy searching by `clientid` or `username` --- apps/emqx/src/emqx_authentication.erl | 8 +-- apps/emqx_authn/src/emqx_authn_api.erl | 22 ++++---- .../src/simple_authn/emqx_authn_mnesia.erl | 55 +++++++++++++++++-- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 18798598e..6dc4c4bbd 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -378,8 +378,8 @@ lookup_user(ChainName, AuthenticatorID, UserID) -> call({lookup_user, ChainName, AuthenticatorID, UserID}). -spec list_users(chain_name(), authenticator_id(), map()) -> {ok, [user_info()]} | {error, term()}. -list_users(ChainName, AuthenticatorID, Params) -> - call({list_users, ChainName, AuthenticatorID, Params}). +list_users(ChainName, AuthenticatorID, FuzzyParams) -> + call({list_users, ChainName, AuthenticatorID, FuzzyParams}). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -476,8 +476,8 @@ handle_call({lookup_user, ChainName, AuthenticatorID, UserID}, _From, State) -> Reply = call_authenticator(ChainName, AuthenticatorID, lookup_user, [UserID]), reply(Reply, State); -handle_call({list_users, ChainName, AuthenticatorID, PageParams}, _From, State) -> - Reply = call_authenticator(ChainName, AuthenticatorID, list_users, [PageParams]), +handle_call({list_users, ChainName, AuthenticatorID, FuzzyParams}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, list_users, [FuzzyParams]), reply(Reply, State); handle_call(Req, _From, State) -> diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index af4c3da2d..623012bee 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -380,7 +380,13 @@ schema("/authentication/:id/users") -> parameters => [ param_auth_id(), {page, mk(integer(), #{in => query, desc => <<"Page Index">>, required => false})}, - {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, required => false})} + {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, required => false})}, + {like_username, mk(binary(), #{ in => query + , desc => <<"Fuzzy search username">> + , required => false})}, + {like_clientid, mk(binary(), #{ in => query + , desc => <<"Fuzzy search clientid">> + , required => false})} ], responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( @@ -638,8 +644,8 @@ listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id : authenticator_users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> add_user(?GLOBAL, AuthenticatorID, UserInfo); -authenticator_users(get, #{bindings := #{id := AuthenticatorID}, query_string := PageParams}) -> - list_users(?GLOBAL, AuthenticatorID, PageParams). +authenticator_users(get, #{bindings := #{id := AuthenticatorID}, query_string := QueryString}) -> + list_users(?GLOBAL, AuthenticatorID, QueryString). authenticator_user(put, #{bindings := #{id := AuthenticatorID, user_id := UserID}, body := UserInfo}) -> @@ -840,13 +846,9 @@ delete_user(ChainName, AuthenticatorID, UserID) -> serialize_error({user_error, Reason}) end. -list_users(ChainName, AuthenticatorID, PageParams) -> - case emqx_authentication:list_users(ChainName, AuthenticatorID, PageParams) of - {ok, Users} -> - {200, Users}; - {error, Reason} -> - serialize_error(Reason) - end. +list_users(ChainName, AuthenticatorID, QueryString) -> + Response = emqx_authentication:list_users(ChainName, AuthenticatorID, QueryString), + emqx_mgmt_util:generate_response(Response). update_config(Path, ConfigRequest) -> emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index b21521d79..7ff9e85ef 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -43,7 +43,9 @@ , list_users/2 ]). --export([format_user_info/1]). +-export([ query/4 + , format_user_info/1 + , group_match_spec/1]). -type user_id_type() :: clientid | username. -type user_group() :: binary(). @@ -63,7 +65,10 @@ -boot_mnesia({mnesia, [boot]}). -define(TAB, ?MODULE). --define(FORMAT_FUN, {?MODULE, format_user_info}). +-define(AUTHN_QSCHEMA, [ {<<"like_username">>, binary} + , {<<"like_clientid">>, binary} + , {<<"user_group">>, binary}]). +-define(QUERY_FUN, {?MODULE, query}). %%------------------------------------------------------------------------------ %% Mnesia bootstrap @@ -219,8 +224,42 @@ lookup_user(UserID, #{user_group := UserGroup}) -> {error, not_found} end. -list_users(PageParams, #{user_group := UserGroup}) -> - {ok, emqx_mgmt_api:paginate(?TAB, group_match_spec(UserGroup), PageParams, ?FORMAT_FUN)}. +list_users(QueryString, #{user_group := UserGroup}) -> + NQueryString = QueryString#{<<"user_group">> => UserGroup}, + emqx_mgmt_api:node_query(node(), NQueryString, ?TAB, ?AUTHN_QSCHEMA, ?QUERY_FUN). + +%%-------------------------------------------------------------------- +%% Query Functions + +query(Tab, {QString, []}, Continuation, Limit) -> + Ms = ms_from_qstring(QString), + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, + fun format_user_info/1); + +query(Tab, {QString, FuzzyQString}, Continuation, Limit) -> + Ms = ms_from_qstring(QString), + FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, + fun format_user_info/1). + +%%-------------------------------------------------------------------- +%% Match funcs + +%% Fuzzy username funcs +fuzzy_filter_fun(Fuzzy) -> + fun(MsRaws) when is_list(MsRaws) -> + lists:filter( fun(E) -> run_fuzzy_filter(E, Fuzzy) end + , MsRaws) + end. + +run_fuzzy_filter(_, []) -> + true; +run_fuzzy_filter( E = #user_info{user_id = {_, UserID}} + , [{username, like, UsernameSubStr} | Fuzzy]) -> + binary:match(UserID, UsernameSubStr) /= nomatch andalso run_fuzzy_filter(E, Fuzzy); +run_fuzzy_filter( E = #user_info{user_id = {_, UserID}} + , [{clientid, like, ClientIDSubStr} | Fuzzy]) -> + binary:match(UserID, ClientIDSubStr) /= nomatch andalso run_fuzzy_filter(E, Fuzzy). %%------------------------------------------------------------------------------ %% Internal functions @@ -352,6 +391,14 @@ to_binary(L) when is_list(L) -> format_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> #{user_id => UserID, is_superuser => IsSuperuser}. +ms_from_qstring(QString) -> + [Ms] = lists:foldl(fun({user_group, '=:=', UserGroup}, AccIn) -> + [group_match_spec(UserGroup) | AccIn]; + (_, AccIn) -> + AccIn + end, [], QString), + Ms. + group_match_spec(UserGroup) -> ets:fun2ms( fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup -> From 990a66e1ad1fd4cde8cf7c208e14d768ada1d7db Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 7 Mar 2022 11:41:26 +0800 Subject: [PATCH 4/6] feat(gateway): gateway authn mnesia rows fuzzy searching by `clientid` or `username` --- .../src/emqx_gateway_api_authn.erl | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 30ea7b050..240ae5028 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -99,7 +99,7 @@ authn(delete, #{bindings := #{name := Name0}}) -> users(get, #{bindings := #{name := Name0}, query_string := Qs}) -> with_authn(Name0, fun(_GwName, #{id := AuthId, chain_name := ChainName}) -> - emqx_authn_api:list_users(ChainName, AuthId, page_pramas(Qs)) + emqx_authn_api:list_users(ChainName, AuthId, parse_qstring(Qs)) end); users(post, #{bindings := #{name := Name0}, body := Body}) -> @@ -145,8 +145,11 @@ import_users(post, #{bindings := #{name := Name0}, %%-------------------------------------------------------------------- %% Utils -page_pramas(Qs) -> - maps:with([<<"page">>, <<"limit">>], Qs). +parse_qstring(Qs) -> + maps:with([ <<"page">> + , <<"limit">> + , <<"like_username">> + , <<"like_clientid">>], Qs). %%-------------------------------------------------------------------- %% Swagger defines @@ -190,7 +193,8 @@ schema("/gateway/:name/authentication/users") -> , get => #{ description => <<"Get the users for the authentication">> , parameters => params_gateway_name_in_path() ++ - params_paging_in_qs() + params_paging_in_qs() ++ + params_fuzzy_in_qs() , responses => ?STANDARD_RESP( #{ 200 => emqx_dashboard_swagger:schema_with_example( @@ -299,6 +303,23 @@ params_paging_in_qs() -> })} ]. +params_fuzzy_in_qs() -> + [{like_username, + mk(binary(), + #{ in => query + , required => false + , desc => <<"Fuzzy search by username">> + , example => <<"username">> + })}, + {like_clientid, + mk(binary(), + #{ in => query + , required => false + , desc => <<"Fuzzy search by clientid">> + , example => <<"clientid">> + })} + ]. + %%-------------------------------------------------------------------- %% schemas From 42df68ab51bfd63de0ad47d0cd151a9d0b6933c4 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 7 Mar 2022 18:23:13 +0800 Subject: [PATCH 5/6] test(authn): fix return type match, api fuzzy searching --- .../test/emqx_authn_mnesia_SUITE.erl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index fbda75a0d..eaee25b1e 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -182,15 +182,22 @@ t_list_users(_) -> fun(U) -> {ok, _} = emqx_authn_mnesia:add_user(U, State) end, Users), - {ok, - #{data := [#{user_id := _}, #{user_id := _}], - meta := #{page := 1, limit := 2, count := 3}}} = emqx_authn_mnesia:list_users( + #{data := [#{is_superuser := false,user_id := <<"u2">>}, + #{is_superuser := false,user_id := <<"u1">>}], + meta := #{page := 1, limit := 2, count := 3}} = emqx_authn_mnesia:list_users( #{<<"page">> => 1, <<"limit">> => 2}, State), - {ok, - #{data := [#{user_id := _}], - meta := #{page := 2, limit := 2, count := 3}}} = emqx_authn_mnesia:list_users( + + #{data := [#{is_superuser := false,user_id := <<"u3">>}], + meta := #{page := 2, limit := 2, count := 3}} = emqx_authn_mnesia:list_users( #{<<"page">> => 2, <<"limit">> => 2}, + State), + + #{data := [#{is_superuser := false,user_id := <<"u3">>}], + meta := #{page := 1, limit := 20, count := 1}} = emqx_authn_mnesia:list_users( + #{ <<"page">> => 1 + , <<"limit">> => 20 + , <<"like_username">> => <<"3">>}, State). t_import_users(_) -> From 7274d775cba570332049840018efce71754c9f78 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 7 Mar 2022 18:52:49 +0800 Subject: [PATCH 6/6] test(authz): api username fuzzy searching --- .../test/emqx_authn_mnesia_SUITE.erl | 6 ++--- .../test/emqx_authz_api_mnesia_SUITE.erl | 26 +++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index eaee25b1e..569f1503a 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -182,13 +182,13 @@ t_list_users(_) -> fun(U) -> {ok, _} = emqx_authn_mnesia:add_user(U, State) end, Users), - #{data := [#{is_superuser := false,user_id := <<"u2">>}, - #{is_superuser := false,user_id := <<"u1">>}], + #{data := [#{is_superuser := false,user_id := _}, + #{is_superuser := false,user_id := _}], meta := #{page := 1, limit := 2, count := 3}} = emqx_authn_mnesia:list_users( #{<<"page">> => 1, <<"limit">> => 2}, State), - #{data := [#{is_superuser := false,user_id := <<"u3">>}], + #{data := [#{is_superuser := false,user_id := _}], meta := #{page := 2, limit := 2, count := 3}} = emqx_authn_mnesia:list_users( #{<<"page">> => 2, <<"limit">> => 2}, State), diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index d618e56ca..3e04e6a8f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -76,21 +76,37 @@ t_api(_) -> request( post , uri(["authorization", "sources", "built-in-database", "username"]) , [?USERNAME_RULES_EXAMPLE]), + {ok, 200, Request1} = request( get , uri(["authorization", "sources", "built-in-database", "username"]) , []), - {ok, 200, Request2} = - request( get - , uri(["authorization", "sources", "built-in-database", "username", "user1"]) - , []), #{<<"data">> := [#{<<"username">> := <<"user1">>, <<"rules">> := Rules1}], <<"meta">> := #{<<"count">> := 1, <<"limit">> := 100, <<"page">> := 1}} = jsx:decode(Request1), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), ?assertEqual(3, length(Rules1)), + {ok, 200, Request1_1} = + request( get + , uri([ "authorization" + , "sources" + , "built-in-database" + , "username?page=1&limit=20&like_username=noexist"]) + , []), + #{<<"data">> := [], + <<"meta">> := #{<<"count">> := 0, + <<"limit">> := 20, + <<"page">> := 1}} = jsx:decode(Request1_1), + + + {ok, 200, Request2} = + request( get + , uri(["authorization", "sources", "built-in-database", "username", "user1"]) + , []), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + + {ok, 204, _} = request( put , uri(["authorization", "sources", "built-in-database", "username", "user1"])