Merge pull request #12509 from SergeTupchiy/EMQX-11770-auth-batch-reorder

feat(emqx_auth): implement API to re-order all authenticators/authz sources
This commit is contained in:
SergeTupchiy 2024-02-14 15:26:04 +02:00 committed by GitHub
commit fa359246c1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 429 additions and 14 deletions

View File

@ -28,6 +28,7 @@
-define(CMD_APPEND, append).
-define(CMD_MOVE, move).
-define(CMD_MERGE, merge).
-define(CMD_REORDER, reorder).
-define(CMD_MOVE_FRONT, front).
-define(CMD_MOVE_REAR, rear).

View File

@ -32,6 +32,8 @@
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
-define(CONFIG, emqx_authn_config).
-define(join(List), lists:join(", ", List)).
% Swagger
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
@ -56,6 +58,7 @@
listener_authenticator/2,
listener_authenticator_status/2,
authenticator_position/2,
authenticators_order/2,
listener_authenticator_position/2,
authenticator_users/2,
authenticator_user/2,
@ -102,7 +105,8 @@ paths() ->
"/authentication/:id/status",
"/authentication/:id/position/:position",
"/authentication/:id/users",
"/authentication/:id/users/:user_id"
"/authentication/:id/users/:user_id",
"/authentication/order"
%% hide listener authn api since 5.1.0
%% "/listeners/:listener_id/authentication",
@ -118,7 +122,8 @@ roots() ->
request_user_create,
request_user_update,
response_user,
response_users
response_users,
request_authn_order
].
fields(request_user_create) ->
@ -137,7 +142,16 @@ fields(response_user) ->
{is_superuser, mk(boolean(), #{default => false, required => false})}
];
fields(response_users) ->
paginated_list_type(ref(response_user)).
paginated_list_type(ref(response_user));
fields(request_authn_order) ->
[
{id,
mk(binary(), #{
desc => ?DESC(param_auth_id),
required => true,
example => "password_based:built_in_database"
})}
].
schema("/authentication") ->
#{
@ -218,7 +232,7 @@ schema("/authentication/:id/status") ->
parameters => [param_auth_id()],
responses => #{
200 => emqx_dashboard_swagger:schema_with_examples(
hoconsc:ref(emqx_authn_schema, "metrics_status_fields"),
ref(emqx_authn_schema, "metrics_status_fields"),
status_metrics_example()
),
404 => error_codes([?NOT_FOUND], <<"Not Found">>),
@ -313,7 +327,7 @@ schema("/listeners/:listener_id/authentication/:id/status") ->
parameters => [param_listener_id(), param_auth_id()],
responses => #{
200 => emqx_dashboard_swagger:schema_with_examples(
hoconsc:ref(emqx_authn_schema, "metrics_status_fields"),
ref(emqx_authn_schema, "metrics_status_fields"),
status_metrics_example()
),
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
@ -530,6 +544,22 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") ->
404 => error_codes([?NOT_FOUND], <<"Not Found">>)
}
}
};
schema("/authentication/order") ->
#{
'operationId' => authenticators_order,
put => #{
tags => ?API_TAGS_GLOBAL,
description => ?DESC(authentication_order_put),
'requestBody' => mk(
hoconsc:array(ref(?MODULE, request_authn_order)),
#{}
),
responses => #{
204 => <<"Authenticators order updated">>,
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
}
}
}.
param_auth_id() ->
@ -670,6 +700,17 @@ listener_authenticator_status(
end
).
authenticators_order(put, #{body := AuthnOrder}) ->
AuthnIdsOrder = [Id || #{<<"id">> := Id} <- AuthnOrder],
case update_config([authentication], {reorder_authenticators, AuthnIdsOrder}) of
{ok, _} ->
{204};
{error, {_PrePostConfigUpdate, ?CONFIG, Reason}} ->
serialize_error(Reason);
{error, Reason} ->
serialize_error(Reason)
end.
authenticator_position(
put,
#{bindings := #{id := AuthenticatorID, position := Position}}
@ -1253,6 +1294,21 @@ serialize_error({unknown_authn_type, Type}) ->
code => <<"BAD_REQUEST">>,
message => binfmt("Unknown type '~p'", [Type])
}};
serialize_error(#{not_found := NotFound, not_reordered := NotReordered}) ->
NotFoundFmt = "Authenticators: ~ts are not found",
NotReorderedFmt = "No positions are specified for authenticators: ~ts",
Msg =
case {NotFound, NotReordered} of
{[_ | _], []} ->
binfmt(NotFoundFmt, [?join(NotFound)]);
{[], [_ | _]} ->
binfmt(NotReorderedFmt, [?join(NotReordered)]);
_ ->
binfmt(NotFoundFmt ++ ", " ++ NotReorderedFmt, [
?join(NotFound), ?join(NotReordered)
])
end,
{400, #{code => <<"BAD_REQUEST">>, message => Msg}};
serialize_error(Reason) ->
{400, #{
code => <<"BAD_REQUEST">>,

View File

@ -135,6 +135,8 @@ do_pre_config_update(_, {move_authenticator, _ChainName, AuthenticatorID, Positi
do_pre_config_update(ConfPath, {merge_authenticators, NewConfig}, OldConfig) ->
MergeConfig = merge_authenticators(OldConfig, NewConfig),
do_pre_config_update(ConfPath, MergeConfig, OldConfig);
do_pre_config_update(_ConfPath, {reorder_authenticators, NewOrder}, OldConfig) ->
reorder_authenticators(NewOrder, OldConfig);
do_pre_config_update(_, OldConfig, OldConfig) ->
{ok, OldConfig};
do_pre_config_update(ConfPath, NewConfig, _OldConfig) ->
@ -194,6 +196,15 @@ do_post_config_update(
_AppEnvs
) ->
emqx_authn_chains:move_authenticator(ChainName, AuthenticatorID, Position);
do_post_config_update(
ConfPath,
{reorder_authenticators, NewOrder},
_NewConfig,
_OldConfig,
_AppEnvs
) ->
ChainName = chain_name(ConfPath),
ok = emqx_authn_chains:reorder_authenticator(ChainName, NewOrder);
do_post_config_update(_, _UpdateReq, OldConfig, OldConfig, _AppEnvs) ->
ok;
do_post_config_update(ConfPath, _UpdateReq, NewConfig0, OldConfig0, _AppEnvs) ->
@ -389,6 +400,24 @@ merge_authenticators(OriginConf0, NewConf0) ->
),
lists:reverse(OriginConf1) ++ NewConf1.
reorder_authenticators(NewOrder, OldConfig) ->
OldConfigWithIds = [{authenticator_id(Auth), Auth} || Auth <- OldConfig],
reorder_authenticators(NewOrder, OldConfigWithIds, [], []).
reorder_authenticators([], [] = _RemConfigWithIds, ReorderedConfig, [] = _NotFoundIds) ->
{ok, lists:reverse(ReorderedConfig)};
reorder_authenticators([], RemConfigWithIds, _ReorderedConfig, NotFoundIds) ->
{error, #{not_found => NotFoundIds, not_reordered => [Id || {Id, _} <- RemConfigWithIds]}};
reorder_authenticators([Id | RemOrder], RemConfigWithIds, ReorderedConfig, NotFoundIds) ->
case lists:keytake(Id, 1, RemConfigWithIds) of
{value, {_Id, Auth}, RemConfigWithIds1} ->
reorder_authenticators(
RemOrder, RemConfigWithIds1, [Auth | ReorderedConfig], NotFoundIds
);
false ->
reorder_authenticators(RemOrder, RemConfigWithIds, ReorderedConfig, [Id | NotFoundIds])
end.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-compile(nowarn_export_all).

View File

@ -36,6 +36,7 @@
lookup/0,
lookup/1,
move/2,
reorder/1,
update/2,
merge/1,
merge_local/2,
@ -64,6 +65,8 @@
maybe_write_files/1
]).
-import(emqx_utils_conv, [bin/1]).
-type default_result() :: allow | deny.
-type authz_result_value() :: #{result := allow | deny, from => _}.
@ -181,6 +184,9 @@ move(Type, Position) ->
?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}
).
reorder(SourcesOrder) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_REORDER, SourcesOrder}).
update({?CMD_REPLACE, Type}, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources});
update({?CMD_DELETE, Type}, Sources) ->
@ -258,6 +264,8 @@ do_pre_config_update({?CMD_REPLACE, Sources}, _OldSources) ->
NSources = lists:map(fun maybe_write_source_files/1, Sources),
ok = check_dup_types(NSources),
NSources;
do_pre_config_update({?CMD_REORDER, NewSourcesOrder}, OldSources) ->
reorder_sources(NewSourcesOrder, OldSources);
do_pre_config_update({Op, Source}, Sources) ->
throw({bad_request, #{op => Op, source => Source, sources => Sources}}).
@ -290,6 +298,16 @@ do_post_config_update(?CONF_KEY_PATH, {{?CMD_DELETE, Type}, _RawNewSource}, _Sou
Front ++ Rear;
do_post_config_update(?CONF_KEY_PATH, {?CMD_REPLACE, _RawNewSources}, Sources) ->
overwrite_entire_sources(Sources);
do_post_config_update(?CONF_KEY_PATH, {?CMD_REORDER, NewSourcesOrder}, _Sources) ->
OldSources = lookup(),
lists:map(
fun(Type) ->
Type1 = type(Type),
{value, Val} = lists:search(fun(S) -> type(S) =:= Type1 end, OldSources),
Val
end,
NewSourcesOrder
);
do_post_config_update(?ROOT_KEY, Conf, Conf) ->
#{sources := Sources} = Conf,
Sources;
@ -729,6 +747,29 @@ type_take(Type, Sources) ->
throw:{not_found_source, Type} -> not_found
end.
reorder_sources(NewOrder, OldSources) ->
NewOrder1 = lists:map(fun type/1, NewOrder),
OldSourcesWithTypes = [{type(Source), Source} || Source <- OldSources],
reorder_sources(NewOrder1, OldSourcesWithTypes, [], []).
reorder_sources([], [] = _RemSourcesWithTypes, ReorderedSources, [] = _NotFoundTypes) ->
lists:reverse(ReorderedSources);
reorder_sources([], RemSourcesWithTypes, _ReorderedSources, NotFoundTypes) ->
{error, #{
not_found => NotFoundTypes, not_reordered => [bin(Type) || {Type, _} <- RemSourcesWithTypes]
}};
reorder_sources([Type | RemOrder], RemSourcesWithTypes, ReorderedSources, NotFoundTypes) ->
case lists:keytake(Type, 1, RemSourcesWithTypes) of
{value, {_Type, Source}, RemSourcesWithTypes1} ->
reorder_sources(
RemOrder, RemSourcesWithTypes1, [Source | ReorderedSources], NotFoundTypes
);
false ->
reorder_sources(RemOrder, RemSourcesWithTypes, ReorderedSources, [
bin(Type) | NotFoundTypes
])
end.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-compile(nowarn_export_all).

View File

@ -27,6 +27,8 @@
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(NOT_FOUND, 'NOT_FOUND').
-define(join(List), lists:join(", ", List)).
-export([
get_raw_sources/0,
get_raw_source/1,
@ -46,6 +48,7 @@
sources/2,
source/2,
source_move/2,
sources_order/2,
aggregate_metrics/1
]).
@ -61,7 +64,8 @@ paths() ->
"/authorization/sources",
"/authorization/sources/:type",
"/authorization/sources/:type/status",
"/authorization/sources/:type/move"
"/authorization/sources/:type/move",
"/authorization/sources/order"
].
fields(sources) ->
@ -77,6 +81,15 @@ fields(position) ->
in => body
}
)}
];
fields(request_sources_order) ->
[
{type,
mk(enum(emqx_authz_schema:source_types()), #{
desc => ?DESC(source_type),
required => true,
example => "file"
})}
].
%%--------------------------------------------------------------------
@ -196,6 +209,22 @@ schema("/authorization/sources/:type/move") ->
404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>)
}
}
};
schema("/authorization/sources/order") ->
#{
'operationId' => sources_order,
put => #{
tags => ?TAGS,
description => ?DESC(authorization_sources_order_put),
'requestBody' => mk(
hoconsc:array(ref(?MODULE, request_sources_order)),
#{}
),
responses => #{
204 => <<"Authorization sources order updated">>,
400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>)
}
}
}.
%%--------------------------------------------------------------------
@ -317,6 +346,30 @@ source_move(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos
end
).
sources_order(put, #{body := AuthzOrder}) ->
SourcesOrder = [Type || #{<<"type">> := Type} <- AuthzOrder],
case emqx_authz:reorder(SourcesOrder) of
{ok, _} ->
{204};
{error, {_PrePostConfUpd, _, #{not_found := NotFound, not_reordered := NotReordered}}} ->
NotFoundFmt = "Authorization sources: ~ts are not found",
NotReorderedFmt = "No positions are specified for authorization sources: ~ts",
Msg =
case {NotFound, NotReordered} of
{[_ | _], []} ->
binfmt(NotFoundFmt, [?join(NotFound)]);
{[], [_ | _]} ->
binfmt(NotReorderedFmt, [?join(NotReordered)]);
_ ->
binfmt(NotFoundFmt ++ ", " ++ NotReorderedFmt, [
?join(NotFound), ?join(NotReordered)
])
end,
{400, #{code => <<"BAD_REQUEST">>, message => Msg}};
{error, Reason} ->
{400, #{code => <<"BAD_REQUEST">>, message => bin(Reason)}}
end.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
@ -556,7 +609,9 @@ position_example() ->
}
}.
bin(Term) -> erlang:iolist_to_binary(io_lib:format("~p", [Term])).
bin(Term) -> binfmt("~p", [Term]).
binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)).
status_metrics_example() ->
#{

View File

@ -124,6 +124,111 @@ t_authenticator_fail(_) ->
t_authenticator_position(_) ->
test_authenticator_position([]).
t_authenticators_reorder(_) ->
AuthenticatorConfs = [
emqx_authn_test_lib:http_example(),
%% Disabling an authenticator must not affect the requested order
(emqx_authn_test_lib:jwt_example())#{enable => false},
emqx_authn_test_lib:built_in_database_example()
],
lists:foreach(
fun(Conf) ->
{ok, 200, _} = request(
post,
uri([?CONF_NS]),
Conf
)
end,
AuthenticatorConfs
),
?assertAuthenticatorsMatch(
[
#{<<"mechanism">> := <<"password_based">>, <<"backend">> := <<"http">>},
#{<<"mechanism">> := <<"jwt">>, <<"enable">> := false},
#{<<"mechanism">> := <<"password_based">>, <<"backend">> := <<"built_in_database">>}
],
[?CONF_NS]
),
OrderUri = uri([?CONF_NS, "order"]),
%% Invalid moves
%% Bad schema
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"not-id">> => <<"password_based:http">>},
#{<<"not-id">> => <<"jwt">>}
]
),
%% Partial order
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"id">> => <<"password_based:http">>},
#{<<"id">> => <<"jwt">>}
]
),
%% Not found authenticators
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"id">> => <<"password_based:http">>},
#{<<"id">> => <<"jwt">>},
#{<<"id">> => <<"password_based:built_in_database">>},
#{<<"id">> => <<"password_based:mongodb">>}
]
),
%% Both partial and not found errors
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"id">> => <<"password_based:http">>},
#{<<"id">> => <<"password_based:built_in_database">>},
#{<<"id">> => <<"password_based:mongodb">>}
]
),
%% Duplicates
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"id">> => <<"password_based:http">>},
#{<<"id">> => <<"password_based:built_in_database">>},
#{<<"id">> => <<"jwt">>},
#{<<"id">> => <<"password_based:http">>}
]
),
%% Valid moves
{ok, 204, _} = request(
put,
OrderUri,
[
#{<<"id">> => <<"password_based:built_in_database">>},
#{<<"id">> => <<"jwt">>},
#{<<"id">> => <<"password_based:http">>}
]
),
?assertAuthenticatorsMatch(
[
#{<<"mechanism">> := <<"password_based">>, <<"backend">> := <<"built_in_database">>},
#{<<"mechanism">> := <<"jwt">>, <<"enable">> := false},
#{<<"mechanism">> := <<"password_based">>, <<"backend">> := <<"http">>}
],
[?CONF_NS]
).
%t_listener_authenticators(_) ->
% test_authenticators(["listeners", ?TCP_DEFAULT]).

View File

@ -29,7 +29,7 @@
-define(PGSQL_HOST, "pgsql").
-define(REDIS_SINGLE_HOST, "redis").
-define(SOURCE_REDIS1, #{
-define(SOURCE_HTTP, #{
<<"type">> => <<"http">>,
<<"enable">> => true,
<<"url">> => <<"https://fake.com:443/acl?username=", ?PH_USERNAME/binary>>,
@ -74,7 +74,7 @@
<<"ssl">> => #{<<"enable">> => false},
<<"query">> => <<"abcb">>
}).
-define(SOURCE_REDIS2, #{
-define(SOURCE_REDIS, #{
<<"type">> => <<"redis">>,
<<"enable">> => true,
<<"servers">> => <<?REDIS_SINGLE_HOST, ",127.0.0.1:6380">>,
@ -188,10 +188,10 @@ t_api(_) ->
{ok, 204, _} = request(post, uri(["authorization", "sources"]), Source)
end
|| Source <- lists:reverse([
?SOURCE_MONGODB, ?SOURCE_MYSQL, ?SOURCE_POSTGRESQL, ?SOURCE_REDIS2, ?SOURCE_FILE
?SOURCE_MONGODB, ?SOURCE_MYSQL, ?SOURCE_POSTGRESQL, ?SOURCE_REDIS, ?SOURCE_FILE
])
],
{ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE_REDIS1),
{ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE_HTTP),
{ok, 200, Result2} = request(get, uri(["authorization", "sources"]), []),
Sources = get_sources(Result2),
@ -211,7 +211,7 @@ t_api(_) ->
{ok, 204, _} = request(
put,
uri(["authorization", "sources", "http"]),
?SOURCE_REDIS1#{<<"enable">> := false}
?SOURCE_HTTP#{<<"enable">> := false}
),
{ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []),
?assertMatch(
@ -338,7 +338,7 @@ t_api(_) ->
{ok, 204, _} = request(
put,
uri(["authorization", "sources", "redis"]),
?SOURCE_REDIS2#{
?SOURCE_REDIS#{
<<"servers">> := [
<<"192.168.1.100:6379">>,
<<"192.168.1.100:6380">>
@ -503,7 +503,7 @@ t_api(_) ->
t_source_move(_) ->
{ok, _} = emqx_authz:update(replace, [
?SOURCE_REDIS1, ?SOURCE_MONGODB, ?SOURCE_MYSQL, ?SOURCE_POSTGRESQL, ?SOURCE_REDIS2
?SOURCE_HTTP, ?SOURCE_MONGODB, ?SOURCE_MYSQL, ?SOURCE_POSTGRESQL, ?SOURCE_REDIS
]),
?assertMatch(
[
@ -582,6 +582,123 @@ t_source_move(_) ->
ok.
t_sources_reorder(_) ->
%% Disabling an auth source must not affect the requested order
MongoDbDisabled = (?SOURCE_MONGODB)#{<<"enable">> => false},
{ok, _} = emqx_authz:update(replace, [
?SOURCE_HTTP, MongoDbDisabled, ?SOURCE_MYSQL, ?SOURCE_POSTGRESQL, ?SOURCE_REDIS
]),
?assertMatch(
[
#{type := http},
#{type := mongodb},
#{type := mysql},
#{type := postgresql},
#{type := redis}
],
emqx_authz:lookup()
),
OrderUri = uri(["authorization", "sources", "order"]),
%% Valid moves
{ok, 204, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>},
#{<<"type">> => <<"mongodb">>}
]
),
?assertMatch(
[
#{type := redis},
#{type := http},
#{type := postgresql},
#{type := mysql},
#{type := mongodb, enable := false}
],
emqx_authz:lookup()
),
%% Invalid moves
%% Bad schema
{ok, 400, _} = request(
put,
OrderUri,
[#{<<"not-type">> => <<"redis">>}]
),
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"unkonw">>},
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>},
#{<<"type">> => <<"mongodb">>}
]
),
%% Partial order
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>}
]
),
%% Not found authenticators
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>},
#{<<"type">> => <<"mongodb">>},
#{<<"type">> => <<"built_in_database">>},
#{<<"type">> => <<"file">>}
]
),
%% Both partial and not found errors
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>},
#{<<"type">> => <<"built_in_database">>}
]
),
%% Duplicates
{ok, 400, _} = request(
put,
OrderUri,
[
#{<<"type">> => <<"redis">>},
#{<<"type">> => <<"http">>},
#{<<"type">> => <<"postgresql">>},
#{<<"type">> => <<"mysql">>},
#{<<"type">> => <<"mongodb">>},
#{<<"type">> => <<"http">>}
]
).
t_aggregate_metrics(_) ->
Metrics = #{
'emqx@node1.emqx.io' => #{

View File

@ -0,0 +1 @@
Implement API to re-order all authenticators / authorization sources.

View File

@ -60,6 +60,11 @@ authentication_post.desc:
authentication_post.label:
"""Create authenticator"""
authentication_order_put.desc:
"""Reorder all authenticators in global authentication chain."""
authentication_order_put.label:
"""Reorder Authenticators"""
is_superuser.desc:
"""Is superuser"""
is_superuser.label:

View File

@ -35,6 +35,11 @@ authorization_sources_type_status_get.desc:
authorization_sources_type_status_get.label:
"""Get a authorization source"""
authorization_sources_order_put.desc:
"""Reorder all authorization sources."""
authorization_sources_order_put.label:
"""Reorder Authorization Sources"""
source.desc:
"""Authorization source"""
source.label: