From 1b9c0825639998a75897c1ddfe26ed8376b6d1d2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 19 Oct 2021 23:32:16 +0200 Subject: [PATCH 1/7] refactor(authn): check authenticator config with provider module mainly two changes: 1. the schema is simplified at root level, per-authenticator checks are done after the type can be identified 2. the config handling part is split out from emqx_authentication module to emqx_authentication_config module --- apps/emqx/src/emqx_authentication.erl | 371 +++--------------- apps/emqx/src/emqx_authentication_config.erl | 323 +++++++++++++++ apps/emqx/src/emqx_broker_sup.erl | 2 +- apps/emqx/src/emqx_config_handler.erl | 9 +- apps/emqx_authn/src/emqx_authn.erl | 47 +++ apps/emqx_authn/src/emqx_authn_api.erl | 21 +- apps/emqx_authn/src/emqx_authn_app.erl | 19 +- apps/emqx_authn/src/emqx_authn_schema.erl | 2 +- .../src/simple_authn/emqx_authn_http.erl | 17 +- 9 files changed, 467 insertions(+), 344 deletions(-) create mode 100644 apps/emqx/src/emqx_authentication_config.erl diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 5f9d72ca7..c96db115f 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -14,33 +14,30 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% @doc Authenticator management API module. +%% Authentication is a core functionality of MQTT, +%% the 'emqx' APP provides APIs for other APPs to implement +%% the authentication callbacks. -module(emqx_authentication). -behaviour(gen_server). --behaviour(hocon_schema). --behaviour(emqx_config_handler). -include("emqx.hrl"). -include("logger.hrl"). --export([ roots/0 - , fields/1 - ]). - --export([ pre_config_update/2 - , post_config_update/4 - ]). - +%% The authentication entrypoint. -export([ authenticate/2 ]). --export([ initialize_authentication/2 ]). - +%% Authenticator manager process start/stop -export([ start_link/0 , stop/0 + , get_providers/0 ]). --export([ register_provider/2 +%% Authenticator management APIs +-export([ initialize_authentication/2 + , register_provider/2 , register_providers/1 , deregister_provider/1 , deregister_providers/1 @@ -56,6 +53,7 @@ , move_authenticator/3 ]). +%% APIs for observer built-in-database -export([ import_users/3 , add_user/3 , delete_user/3 @@ -64,8 +62,6 @@ , list_users/2 ]). --export([ generate_id/1 ]). - %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -75,6 +71,20 @@ , code_change/3 ]). +%% utility functions +-export([ authenticator_id/1 + ]). + +%% proxy callback +-export([ pre_config_update/2 + , post_config_update/4 + ]). + +-export_type([ authenticator_id/0 + , position/0 + , chain_name/0 + ]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -88,10 +98,6 @@ -type chain_name() :: atom(). -type authenticator_id() :: binary(). -type position() :: top | bottom | {before, authenticator_id()}. --type update_request() :: {create_authenticator, chain_name(), map()} - | {delete_authenticator, chain_name(), authenticator_id()} - | {update_authenticator, chain_name(), authenticator_id(), map()} - | {move_authenticator, chain_name(), authenticator_id(), position()}. -type authn_type() :: atom() | {atom(), atom()}. -type provider() :: module(). @@ -103,8 +109,7 @@ enable := boolean(), state := map()}. - --type config() :: #{atom() => term()}. +-type config() :: emqx_authentication_config:config(). -type state() :: #{atom() => term()}. -type extra() :: #{is_superuser := boolean(), atom() => term()}. @@ -173,117 +178,6 @@ , list_users/1 ]). -%%------------------------------------------------------------------------------ -%% Hocon Schema -%%------------------------------------------------------------------------------ - -roots() -> [{authentication, fun authentication/1}]. - -fields(_) -> []. - -authentication(type) -> - {ok, Refs} = get_refs(), - hoconsc:union([hoconsc:array(hoconsc:union(Refs)) | Refs]); -authentication(default) -> []; -authentication(_) -> undefined. - -%%------------------------------------------------------------------------------ -%% Callbacks of config handler -%%------------------------------------------------------------------------------ - --spec pre_config_update(update_request(), emqx_config:raw_config()) - -> {ok, map() | list()} | {error, term()}. -pre_config_update(UpdateReq, OldConfig) -> - case do_pre_config_update(UpdateReq, to_list(OldConfig)) of - {error, Reason} -> {error, Reason}; - {ok, NewConfig} -> {ok, may_to_map(NewConfig)} - end. - -do_pre_config_update({create_authenticator, ChainName, Config}, OldConfig) -> - try - CertsDir = certs_dir([to_bin(ChainName), generate_id(Config)]), - NConfig = convert_certs(CertsDir, Config), - {ok, OldConfig ++ [NConfig]} - catch - error:{save_cert_to_file, _} = Reason -> - {error, Reason}; - error:{missing_parameter, _} = Reason -> - {error, Reason} - end; -do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> - NewConfig = lists:filter(fun(OldConfig0) -> - AuthenticatorID =/= generate_id(OldConfig0) - end, OldConfig), - {ok, NewConfig}; -do_pre_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, OldConfig) -> - try - CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), - NewConfig = lists:map( - fun(OldConfig0) -> - case AuthenticatorID =:= generate_id(OldConfig0) of - true -> convert_certs(CertsDir, Config, OldConfig0); - false -> OldConfig0 - end - end, OldConfig), - {ok, NewConfig} - catch - error:{save_cert_to_file, _} = Reason -> - {error, Reason}; - error:{missing_parameter, _} = Reason -> - {error, Reason} - end; -do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> - case split_by_id(AuthenticatorID, OldConfig) of - {error, Reason} -> {error, Reason}; - {ok, Part1, [Found | Part2]} -> - case Position of - top -> - {ok, [Found | Part1] ++ Part2}; - bottom -> - {ok, Part1 ++ Part2 ++ [Found]}; - {before, Before} -> - case split_by_id(Before, Part1 ++ Part2) of - {error, Reason} -> - {error, Reason}; - {ok, NPart1, [NFound | NPart2]} -> - {ok, NPart1 ++ [Found, NFound | NPart2]} - end - end - end. - --spec post_config_update(update_request(), map() | list(), emqx_config:raw_config(), emqx_config:app_envs()) - -> ok | {ok, map()} | {error, term()}. -post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs) -> - do_post_config_update(UpdateReq, check_config(to_list(NewConfig)), OldConfig, AppEnvs). - -do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _OldConfig, _AppEnvs) -> - NConfig = check_config(Config), - _ = create_chain(ChainName), - create_authenticator(ChainName, NConfig); - -do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, OldConfig, _AppEnvs) -> - case delete_authenticator(ChainName, AuthenticatorID) of - ok -> - [Config] = [Config0 || Config0 <- to_list(OldConfig), AuthenticatorID == generate_id(Config0)], - CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), - clear_certs(CertsDir, Config), - ok; - {error, Reason} -> - {error, Reason} - end; - -do_post_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, _NewConfig, _OldConfig, _AppEnvs) -> - NConfig = check_config(Config), - update_authenticator(ChainName, AuthenticatorID, NConfig); - -do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> - move_authenticator(ChainName, AuthenticatorID, Position). - -check_config(Config) -> - #{authentication := CheckedConfig} = - hocon_schema:check_plain(?MODULE, #{<<"authentication">> => Config}, #{atom_key => true}), - CheckedConfig. - %%------------------------------------------------------------------------------ %% Authenticate %%------------------------------------------------------------------------------ @@ -338,12 +232,30 @@ get_enabled(Authenticators) -> %% APIs %%------------------------------------------------------------------------------ --spec initialize_authentication(chain_name(), [#{binary() => term()}]) -> ok. -initialize_authentication(_, []) -> - ok; +pre_config_update(UpdateReq, OldConfig) -> + emqx_authentication_config:pre_config_update(UpdateReq, OldConfig). + +post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs) -> + emqx_authentication_config:post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs). + +%% @doc Get all registered authentication providers. +get_providers() -> + call(get_providers). + +%% @doc Get authenticator identifier from its config. +%% The authenticator config must contain a 'mechanism' key +%% and maybe a 'backend' key. +%% This function works with both parsed (atom keys) and raw (binary keys) +%% configurations. +authenticator_id(Config) -> + emqx_authentication_config:authenticator_id(Config). + +%% @doc Call this API to initialize authenticators implemented in another APP. +-spec initialize_authentication(chain_name(), config()) -> ok. +initialize_authentication(_, []) -> ok; initialize_authentication(ChainName, AuthenticatorsConfig) -> _ = create_chain(ChainName), - CheckedConfig = check_config(to_list(AuthenticatorsConfig)), + CheckedConfig = to_list(AuthenticatorsConfig), lists:foreach(fun(AuthenticatorConfig) -> case create_authenticator(ChainName, AuthenticatorConfig) of {ok, _} -> @@ -351,7 +263,7 @@ initialize_authentication(ChainName, AuthenticatorsConfig) -> {error, Reason} -> ?SLOG(error, #{ msg => "failed_to_create_authenticator", - authenticator => generate_id(AuthenticatorConfig), + authenticator => authenticator_id(AuthenticatorConfig), reason => Reason }) end @@ -365,10 +277,6 @@ start_link() -> stop() -> gen_server:stop(?MODULE). --spec get_refs() -> {ok, Refs} when Refs :: [{authn_type(), module()}]. -get_refs() -> - call(get_refs). - %% @doc Register authentication providers. %% A provider is a tuple of `AuthNType' the module which implements %% the authenticator callbacks. @@ -472,20 +380,6 @@ lookup_user(ChainName, AuthenticatorID, UserID) -> list_users(ChainName, AuthenticatorID) -> call({list_users, ChainName, AuthenticatorID}). --spec generate_id(config()) -> authenticator_id(). -generate_id(#{mechanism := Mechanism0, backend := Backend0}) -> - Mechanism = to_bin(Mechanism0), - Backend = to_bin(Backend0), - <>; -generate_id(#{mechanism := Mechanism}) -> - to_bin(Mechanism); -generate_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> - <>; -generate_id(#{<<"mechanism">> := Mechanism}) -> - Mechanism; -generate_id(_) -> - error({missing_parameter, mechanism}). - %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- @@ -498,6 +392,8 @@ init(_Opts) -> ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), {ok, #{hooked => false, providers => #{}}}. +handle_call(get_providers, _From, #{providers := Providers} = State) -> + reply(Providers, State); handle_call({register_providers, Providers}, _From, #{providers := Reg0} = State) -> case lists:filter(fun({T, _}) -> maps:is_key(T, Reg0) end, Providers) of @@ -513,12 +409,6 @@ handle_call({register_providers, Providers}, _From, handle_call({deregister_providers, AuthNTypes}, _From, #{providers := Providers} = State) -> reply(ok, State#{providers := maps:without(AuthNTypes, Providers)}); -handle_call(get_refs, _From, #{providers := Providers} = State) -> - Refs = lists:foldl(fun({_, Provider}, Acc) -> - Acc ++ Provider:refs() - end, [], maps:to_list(Providers)), - reply({ok, Refs}, State); - handle_call({create_chain, Name}, _From, State) -> case ets:member(?CHAINS_TAB, Name) of true -> @@ -549,9 +439,9 @@ handle_call({lookup_chain, Name}, _From, State) -> end; handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Providers} = State) -> - UpdateFun = + UpdateFun = fun(#chain{authenticators = Authenticators} = Chain) -> - AuthenticatorID = generate_id(Config), + AuthenticatorID = authenticator_id(Config), case lists:keymember(AuthenticatorID, #authenticator.id, Authenticators) of true -> {error, {already_exists, {authenticator, AuthenticatorID}}}; @@ -570,7 +460,7 @@ handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Pro reply(Reply, maybe_hook(State)); handle_call({delete_authenticator, ChainName, AuthenticatorID}, _From, State) -> - UpdateFun = + UpdateFun = fun(#chain{authenticators = Authenticators} = Chain) -> case lists:keytake(AuthenticatorID, #authenticator.id, Authenticators) of false -> @@ -592,7 +482,7 @@ handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, S {error, {not_found, {authenticator, AuthenticatorID}}}; #authenticator{provider = Provider, state = #{version := Version} = ST} = Authenticator -> - case AuthenticatorID =:= generate_id(Config) of + case AuthenticatorID =:= authenticator_id(Config) of true -> Unique = unique(ChainName, AuthenticatorID, Version), case Provider:update(Config#{'_unique' => Unique}, ST) of @@ -614,7 +504,7 @@ handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, S reply(Reply, State); handle_call({move_authenticator, ChainName, AuthenticatorID, Position}, _From, State) -> - UpdateFun = + UpdateFun = fun(#chain{authenticators = Authenticators} = Chain) -> case do_move_authenticator(AuthenticatorID, Authenticators, Position) of {ok, NAuthenticators} -> @@ -663,9 +553,9 @@ handle_info(Info, State) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. -terminate(_Reason, _State) -> - emqx_config_handler:remove_handler([authentication]), - emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), +terminate(Reason, _State) -> + ?SLOG(error, #{msg => "emqx_authentication_terminating", + reason => Reason}), ok. code_change(_OldVsn, State, _Extra) -> @@ -674,128 +564,6 @@ code_change(_OldVsn, State, _Extra) -> reply(Reply, State) -> {reply, Reply, State}. -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -certs_dir(Dirs) when is_list(Dirs) -> - to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn"] ++ Dirs)). - -convert_certs(CertsDir, Config) -> - case maps:get(<<"ssl">>, Config, undefined) of - undefined -> - Config; - SSLOpts -> - NSSLOPts = lists:foldl(fun(K, Acc) -> - case maps:get(K, Acc, undefined) of - undefined -> Acc; - PemBin -> - CertFile = generate_filename(CertsDir, K), - ok = save_cert_to_file(CertFile, PemBin), - Acc#{K => CertFile} - end - end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), - Config#{<<"ssl">> => NSSLOPts} - end. - -convert_certs(CertsDir, NewConfig, OldConfig) -> - case maps:get(<<"ssl">>, NewConfig, undefined) of - undefined -> - NewConfig; - NewSSLOpts -> - OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), - Diff = diff_certs(NewSSLOpts, OldSSLOpts), - NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> - Acc#{K => maps:get(K, OldSSLOpts)}; - ({_, K}, Acc) -> - CertFile = generate_filename(CertsDir, K), - ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), - Acc#{K => CertFile} - end, NewSSLOpts, Diff), - NewConfig#{<<"ssl">> => NSSLOpts} - end. - -clear_certs(CertsDir, Config) -> - case maps:get(<<"ssl">>, Config, undefined) of - undefined -> - ok; - SSLOpts -> - lists:foreach( - fun({_, Filename}) -> - _ = file:delete(filename:join([CertsDir, Filename])) - end, - maps:to_list(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], SSLOpts))) - end. - -save_cert_to_file(Filename, PemBin) -> - case public_key:pem_decode(PemBin) =/= [] of - true -> - case filelib:ensure_dir(Filename) of - ok -> - case file:write_file(Filename, PemBin) of - ok -> ok; - {error, Reason} -> error({save_cert_to_file, {write_file, Reason}}) - end; - {error, Reason} -> - error({save_cert_to_file, {ensure_dir, Reason}}) - end; - false -> - error({save_cert_to_file, invalid_certificate}) - end. - -generate_filename(CertsDir, Key) -> - Prefix = case Key of - <<"keyfile">> -> "key-"; - <<"certfile">> -> "cert-"; - <<"cacertfile">> -> "cacert-" - end, - to_bin(filename:join([CertsDir, Prefix ++ emqx_misc:gen_id() ++ ".pem"])). - -diff_certs(NewSSLOpts, OldSSLOpts) -> - Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], - CertPems = maps:with(Keys, NewSSLOpts), - CertFiles = maps:with(Keys, OldSSLOpts), - Diff = lists:foldl(fun({K, CertFile}, Acc) -> - case maps:find(K, CertPems) of - error -> Acc; - {ok, PemBin1} -> - {ok, PemBin2} = file:read_file(CertFile), - case diff_cert(PemBin1, PemBin2) of - true -> - [{changed, K} | Acc]; - false -> - [{identical, K} | Acc] - end - end - end, - [], maps:to_list(CertFiles)), - Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(CertFiles), CertPems))], - Diff ++ Added. - -diff_cert(Pem1, Pem2) -> - cal_md5_for_cert(Pem1) =/= cal_md5_for_cert(Pem2). - -cal_md5_for_cert(Pem) -> - crypto:hash(md5, term_to_binary(public_key:pem_decode(Pem))). - -split_by_id(ID, AuthenticatorsConfig) -> - case lists:foldl( - fun(C, {P1, P2, F0}) -> - F = case ID =:= generate_id(C) of - true -> true; - false -> F0 - end, - case F of - false -> {[C | P1], P2, F}; - true -> {P1, [C | P2], F} - end - end, {[], [], false}, AuthenticatorsConfig) of - {_, _, false} -> - {error, {not_found, {authenticator, ID}}}; - {Part1, Part2, true} -> - {ok, lists:reverse(Part1), lists:reverse(Part2)} - end. - global_chain(mqtt) -> 'mqtt:global'; global_chain('mqtt-sn') -> @@ -942,22 +710,9 @@ authn_type(#{mechanism := Mechanism, backend := Backend}) -> authn_type(#{mechanism := Mechanism}) -> Mechanism. -may_to_map([L]) -> - L; -may_to_map(L) -> - L. - -to_list(undefined) -> - []; -to_list(M) when M =:= #{} -> - []; -to_list(M) when is_map(M) -> - [M]; -to_list(L) when is_list(L) -> - L. - -to_bin(B) when is_binary(B) -> B; -to_bin(L) when is_list(L) -> list_to_binary(L); -to_bin(A) when is_atom(A) -> atom_to_binary(A). +to_list(undefined) -> []; +to_list(M) when M =:= #{} -> []; +to_list(M) when is_map(M) -> [M]; +to_list(L) when is_list(L) -> L. call(Call) -> gen_server:call(?MODULE, Call, infinity). diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl new file mode 100644 index 000000000..526d9d6ff --- /dev/null +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -0,0 +1,323 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Authenticator configuration management module. +-module(emqx_authentication_config). + +-behaviour(emqx_config_handler). + +-export([ pre_config_update/2 + , post_config_update/4 + ]). + +-export([ authenticator_id/1 + , authn_type/1 + ]). + +-export_type([config/0]). + +-include("logger.hrl"). + +-type parsed_config() :: #{mechanism := atom(), + backend => atom(), + atom() => term()}. +-type raw_config() :: #{binary() => term()}. +-type config() :: parsed_config() | raw_config(). + +-type authenticator_id() :: emqx_authentication:authenticator_id(). +-type position() :: emqx_authentication:position(). +-type chain_name() :: emqx_authentication:chain_name(). +-type update_request() :: {create_authenticator, chain_name(), map()} + | {delete_authenticator, chain_name(), authenticator_id()} + | {update_authenticator, chain_name(), authenticator_id(), map()} + | {move_authenticator, chain_name(), authenticator_id(), position()}. + +%%------------------------------------------------------------------------------ +%% Callbacks of config handler +%%------------------------------------------------------------------------------ + +-spec pre_config_update(update_request(), emqx_config:raw_config()) + -> {ok, map() | list()} | {error, term()}. +pre_config_update(UpdateReq, OldConfig) -> + case do_pre_config_update(UpdateReq, to_list(OldConfig)) of + {error, Reason} -> {error, Reason}; + {ok, NewConfig} -> {ok, return_map(NewConfig)} + end. + +do_pre_config_update({create_authenticator, ChainName, Config}, OldConfig) -> + try + CertsDir = certs_dir([to_bin(ChainName), authenticator_id(Config)]), + NConfig = convert_certs(CertsDir, Config), + {ok, OldConfig ++ [NConfig]} + catch + error:{save_cert_to_file, _} = Reason -> + {error, Reason}; + error:{missing_parameter, _} = Reason -> + {error, Reason} + end; +do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> + NewConfig = lists:filter(fun(OldConfig0) -> + AuthenticatorID =/= authenticator_id(OldConfig0) + end, OldConfig), + {ok, NewConfig}; +do_pre_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, OldConfig) -> + try + CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), + NewConfig = lists:map( + fun(OldConfig0) -> + case AuthenticatorID =:= authenticator_id(OldConfig0) of + true -> convert_certs(CertsDir, Config, OldConfig0); + false -> OldConfig0 + end + end, OldConfig), + {ok, NewConfig} + catch + error:{save_cert_to_file, _} = Reason -> + {error, Reason}; + error:{missing_parameter, _} = Reason -> + {error, Reason} + end; +do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> + case split_by_id(AuthenticatorID, OldConfig) of + {error, Reason} -> {error, Reason}; + {ok, Part1, [Found | Part2]} -> + case Position of + top -> + {ok, [Found | Part1] ++ Part2}; + bottom -> + {ok, Part1 ++ Part2 ++ [Found]}; + {before, Before} -> + case split_by_id(Before, Part1 ++ Part2) of + {error, Reason} -> + {error, Reason}; + {ok, NPart1, [NFound | NPart2]} -> + {ok, NPart1 ++ [Found, NFound | NPart2]} + end + end + end. + +-spec post_config_update(update_request(), map() | list(), emqx_config:raw_config(), emqx_config:app_envs()) + -> ok | {ok, map()} | {error, term()}. +post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs) -> + do_post_config_update(UpdateReq, check_configs(to_list(NewConfig)), OldConfig, AppEnvs). + +do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _OldConfig, _AppEnvs) -> + NConfig = check_config(Config), + _ = emqx_authentication:create_chain(ChainName), + emqx_authentication:create_authenticator(ChainName, NConfig); +do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, OldConfig, _AppEnvs) -> + case emqx_authentication:delete_authenticator(ChainName, AuthenticatorID) of + ok -> + [Config] = [Config0 || Config0 <- to_list(OldConfig), AuthenticatorID == authenticator_id(Config0)], + CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), + clear_certs(CertsDir, Config), + ok; + {error, Reason} -> + {error, Reason} + end; +do_post_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, _NewConfig, _OldConfig, _AppEnvs) -> + NConfig = check_config(Config), + emqx_authentication:update_authenticator(ChainName, AuthenticatorID, NConfig); +do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> + emqx_authentication:move_authenticator(ChainName, AuthenticatorID, Position). + +check_config(Config) -> + [Checked] = check_configs([Config]), + Checked. + +check_configs(Configs) -> + Providers = emqx_authentication:get_providers(), + lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs). + +do_check_conifg(Config, Providers) -> + Type = authn_type(Config), + case maps:get(Type, Providers, false) of + false -> + ?SLOG(warning, #{msg => "unknown_authn_type", + type => Type, + providers => Providers}), + throw(unknown_authn_type); + Module -> + %% TODO: check if Module:check_config/1 is exported + %% so we do not force all providers to implement hocon schema + try hocon_schema:check_plain(Module, #{<<"config">> => Config}, + #{atom_key => true}) of + #{config := Result} -> + Result + catch + C : E : S -> + ?SLOG(warning, #{msg => "failed_to_check_config", config => Config}), + erlang:raise(C, E, S) + end + end. + +return_map([L]) -> L; +return_map(L) -> L. + +to_list(undefined) -> []; +to_list(M) when M =:= #{} -> []; +to_list(M) when is_map(M) -> [M]; +to_list(L) when is_list(L) -> L. + +certs_dir(Dirs) when is_list(Dirs) -> + to_bin(filename:join([emqx:get_config([node, data_dir]), "certs", "authn"] ++ Dirs)). + +convert_certs(CertsDir, Config) -> + case maps:get(<<"ssl">>, Config, undefined) of + undefined -> + Config; + SSLOpts -> + NSSLOPts = lists:foldl(fun(K, Acc) -> + case maps:get(K, Acc, undefined) of + undefined -> Acc; + PemBin -> + CertFile = generate_filename(CertsDir, K), + ok = save_cert_to_file(CertFile, PemBin), + Acc#{K => CertFile} + end + end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), + Config#{<<"ssl">> => NSSLOPts} + end. + +convert_certs(CertsDir, NewConfig, OldConfig) -> + case maps:get(<<"ssl">>, NewConfig, undefined) of + undefined -> + NewConfig; + NewSSLOpts -> + OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), + Diff = diff_certs(NewSSLOpts, OldSSLOpts), + NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> + Acc#{K => maps:get(K, OldSSLOpts)}; + ({_, K}, Acc) -> + CertFile = generate_filename(CertsDir, K), + ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), + Acc#{K => CertFile} + end, NewSSLOpts, Diff), + NewConfig#{<<"ssl">> => NSSLOpts} + end. + +clear_certs(CertsDir, Config) -> + case maps:get(<<"ssl">>, Config, undefined) of + undefined -> + ok; + SSLOpts -> + lists:foreach( + fun({_, Filename}) -> + _ = file:delete(filename:join([CertsDir, Filename])) + end, + maps:to_list(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], SSLOpts))) + end. + +save_cert_to_file(Filename, PemBin) -> + case public_key:pem_decode(PemBin) =/= [] of + true -> + case filelib:ensure_dir(Filename) of + ok -> + case file:write_file(Filename, PemBin) of + ok -> ok; + {error, Reason} -> error({save_cert_to_file, {write_file, Reason}}) + end; + {error, Reason} -> + error({save_cert_to_file, {ensure_dir, Reason}}) + end; + false -> + error({save_cert_to_file, invalid_certificate}) + end. + +generate_filename(CertsDir, Key) -> + Prefix = case Key of + <<"keyfile">> -> "key-"; + <<"certfile">> -> "cert-"; + <<"cacertfile">> -> "cacert-" + end, + to_bin(filename:join([CertsDir, Prefix ++ emqx_misc:gen_id() ++ ".pem"])). + +diff_certs(NewSSLOpts, OldSSLOpts) -> + Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], + CertPems = maps:with(Keys, NewSSLOpts), + CertFiles = maps:with(Keys, OldSSLOpts), + Diff = lists:foldl(fun({K, CertFile}, Acc) -> + case maps:find(K, CertPems) of + error -> Acc; + {ok, PemBin1} -> + {ok, PemBin2} = file:read_file(CertFile), + case diff_cert(PemBin1, PemBin2) of + true -> + [{changed, K} | Acc]; + false -> + [{identical, K} | Acc] + end + end + end, + [], maps:to_list(CertFiles)), + Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(CertFiles), CertPems))], + Diff ++ Added. + +diff_cert(Pem1, Pem2) -> + cal_md5_for_cert(Pem1) =/= cal_md5_for_cert(Pem2). + +cal_md5_for_cert(Pem) -> + crypto:hash(md5, term_to_binary(public_key:pem_decode(Pem))). + +split_by_id(ID, AuthenticatorsConfig) -> + case lists:foldl( + fun(C, {P1, P2, F0}) -> + F = case ID =:= authenticator_id(C) of + true -> true; + false -> F0 + end, + case F of + false -> {[C | P1], P2, F}; + true -> {P1, [C | P2], F} + end + end, {[], [], false}, AuthenticatorsConfig) of + {_, _, false} -> + {error, {not_found, {authenticator, ID}}}; + {Part1, Part2, true} -> + {ok, lists:reverse(Part1), lists:reverse(Part2)} + end. + +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L); +to_bin(A) when is_atom(A) -> atom_to_binary(A). + +%% @doc Make an authenticator ID from authenticator's config. +%% The authenticator config must contain a 'mechanism' key +%% and maybe a 'backend' key. +%% This function works with both parsed (atom keys) and raw (binary keys) +%% configurations. +-spec authenticator_id(config()) -> authenticator_id(). +authenticator_id(#{mechanism := Mechanism0, backend := Backend0}) -> + Mechanism = to_bin(Mechanism0), + Backend = to_bin(Backend0), + <>; +authenticator_id(#{mechanism := Mechanism}) -> + to_bin(Mechanism); +authenticator_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> + <>; +authenticator_id(#{<<"mechanism">> := Mechanism}) -> + Mechanism; +authenticator_id(C) -> + error({missing_parameter, mechanism, C}). + +%% @doc Make the authentication type. +authn_type(#{mechanism := M, backend := B}) -> {atom(M), atom(B)}; +authn_type(#{mechanism := M}) -> atom(M); +authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)}; +authn_type(#{<<"mechanism">> := M}) -> atom(M). + +atom(Bin) -> + binary_to_existing_atom(Bin, utf8). diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index 761537e57..ae917110e 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -50,7 +50,7 @@ init([]) -> shutdown => infinity, type => supervisor, modules => [emqx_authentication_sup]}, - + %% Broker helper Helper = #{id => helper, start => {emqx_broker_helper, start_link, []}, diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index f53b71901..297f1147d 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -71,15 +71,15 @@ stop() -> update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> %% force covert the path to a list of atoms, as there maybe some wildcard names/ids in the path AtomKeyPath = [atom(Key) || Key <- ConfKeyPath], - gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}). + gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}, infinity). -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> - gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}). + gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}, infinity). -spec remove_handler(emqx_config:config_key_path()) -> ok. remove_handler(ConfKeyPath) -> - gen_server:call(?MODULE, {remove_handler, ConfKeyPath}). + gen_server:call(?MODULE, {remove_handler, ConfKeyPath}, infinity). %%============================================================================ @@ -247,7 +247,8 @@ call_post_config_update(Handlers, OldConf, NewConf, AppEnvs, UpdateReq, Result) true -> case HandlerName:post_config_update(UpdateReq, NewConf, OldConf, AppEnvs) of ok -> {ok, Result}; - {ok, Result1} -> {ok, Result#{HandlerName => Result1}}; + {ok, Result1} -> + {ok, Result#{HandlerName => Result1}}; {error, Reason} -> {error, {post_config_update, HandlerName, Reason}} end; false -> {ok, Result} diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 3ab05e6b0..1b0d12fd2 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -15,3 +15,50 @@ %%-------------------------------------------------------------------- -module(emqx_authn). + +-export([ providers/0 + , check_config/1 + , check_config/2 + , check_configs/1 + ]). + +providers() -> + [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} + , {{'password-based', mysql}, emqx_authn_mysql} + , {{'password-based', postgresql}, emqx_authn_pgsql} + , {{'password-based', mongodb}, emqx_authn_mongodb} + , {{'password-based', redis}, emqx_authn_redis} + , {{'password-based', 'http-server'}, emqx_authn_http} + , {{'password-based', 'http'}, emqx_authn_http} %% TODO: resolve one + , {jwt, emqx_authn_jwt} + , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} + ]. + +check_configs([]) -> []; +check_configs([Config | Configs]) -> + [check_config(Config) | check_configs(Configs)]. + +check_config(Config) -> + check_config(Config, #{}). + +check_config(Config, Opts) -> + case do_check_config(Config, Opts) of + #{config := Checked} -> Checked; + #{<<"config">> := WithDefaults} -> WithDefaults + end. + +do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> + Key = case maps:get(<<"backend">>, Config, false) of + false -> atom(Mec); + Backend -> {atom(Mec), atom(Backend)} + end, + case lists:keyfind(Key, 1, providers()) of + false -> + throw({unknown_handler, Key}); + {_, Provider} -> + hocon_schema:check_plain(Provider, #{<<"config">> => Config}, + Opts#{atom_key => true}) + end. + +atom(Bin) -> + binary_to_existing_atom(Bin, utf8). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 0f89c68d8..93df83829 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -313,7 +313,7 @@ create_authenticator_api_spec() -> }, <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } + } }. create_authenticator_api_spec2() -> @@ -1852,7 +1852,7 @@ create_authenticator(ConfKeyPath, ChainName, Config) -> list_authenticators(ConfKeyPath) -> AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), - NAuthenticators = [maps:put(id, ?AUTHN:generate_id(AuthenticatorConfig), convert_certs(AuthenticatorConfig)) + NAuthenticators = [maps:put(id, ?AUTHN:authenticator_id(AuthenticatorConfig), convert_certs(AuthenticatorConfig)) || AuthenticatorConfig <- AuthenticatorsConfig], {200, NAuthenticators}. @@ -1961,19 +1961,18 @@ update_config(Path, ConfigRequest) -> get_raw_config_with_defaults(ConfKeyPath) -> NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], RawConfig = emqx_map_lib:deep_get(NConfKeyPath, emqx_config:get_raw([]), []), - to_list(fill_defaults(RawConfig)). + ensure_list(fill_defaults(RawConfig)). find_config(AuthenticatorID, AuthenticatorsConfig) -> - case [AC || AC <- to_list(AuthenticatorsConfig), AuthenticatorID =:= ?AUTHN:generate_id(AC)] of + case [AC || AC <- ensure_list(AuthenticatorsConfig), AuthenticatorID =:= ?AUTHN:authenticator_id(AC)] of [] -> {error, {not_found, {authenticator, AuthenticatorID}}}; [AuthenticatorConfig] -> {ok, AuthenticatorConfig} end. +fill_defaults(Configs) when is_list(Configs) -> + lists:map(fun fill_defaults/1, Configs); fill_defaults(Config) -> - #{<<"authentication">> := CheckedConfig} = - hocon_schema:check_plain(?AUTHN, #{<<"authentication">> => Config}, - #{only_fill_defaults => true}), - CheckedConfig. + emqx_authn:check_config(Config, #{only_fill_defaults => true}). convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> NSSLOpts = lists:foldl(fun(K, Acc) -> @@ -2063,10 +2062,8 @@ parse_position(<<"before:", Before/binary>>) -> parse_position(_) -> {error, {invalid_parameter, position}}. -to_list(M) when is_map(M) -> - [M]; -to_list(L) when is_list(L) -> - L. +ensure_list(M) when is_map(M) -> [M]; +ensure_list(L) when is_list(L) -> L. to_atom(B) when is_binary(B) -> binary_to_atom(B); diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index d297c9042..9284493bf 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -32,7 +32,7 @@ start(_StartType, _StartArgs) -> ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), - ok = ?AUTHN:register_providers(providers()), + ok = ?AUTHN:register_providers(emqx_authn:providers()), ok = initialize(), {ok, Sup}. @@ -45,21 +45,12 @@ stop(_State) -> %%------------------------------------------------------------------------------ initialize() -> - ?AUTHN:initialize_authentication(?GLOBAL, emqx:get_raw_config([authentication], [])), + RawConfigs = emqx:get_raw_config([authentication], []), + Config = emqx_authn:check_configs(RawConfigs), + ?AUTHN:initialize_authentication(?GLOBAL, Config), lists:foreach(fun({ListenerID, ListenerConfig}) -> ?AUTHN:initialize_authentication(ListenerID, maps:get(authentication, ListenerConfig, [])) end, emqx_listeners:list()). provider_types() -> - lists:map(fun({Type, _Module}) -> Type end, providers()). - -providers() -> - [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} - , {{'password-based', mysql}, emqx_authn_mysql} - , {{'password-based', postgresql}, emqx_authn_pgsql} - , {{'password-based', mongodb}, emqx_authn_mongodb} - , {{'password-based', redis}, emqx_authn_redis} - , {{'password-based', 'http-server'}, emqx_authn_http} - , {jwt, emqx_authn_jwt} - , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} - ]. + lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 23e412088..ca5fe3cc7 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,7 +22,7 @@ ]). common_fields() -> - [ {enable, fun enable/1} + [ {enable, fun enable/1} ]. enable(type) -> boolean(); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 0c5d3f7f7..5ee7a8db8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -62,7 +62,7 @@ fields(post) -> common_fields() -> [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['http-server']}} + , {backend, {enum, ['http-server', 'http']}} %% TODO: delete http , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} @@ -78,6 +78,7 @@ validations() -> url(type) -> binary(); url(validator) -> [fun check_url/1]; +url(nullable) -> false; url(_) -> undefined. headers(type) -> map(); @@ -214,11 +215,19 @@ transform_header_name(Headers) -> end, #{}, Headers). check_ssl_opts(Conf) -> - emqx_connector_http:check_ssl_opts("url", Conf). + case parse_url(hocon_schema:get_value("config.url", Conf)) of + #{scheme := https} -> + case hocon_schema:get_value("config.ssl.enable", Conf) of + true -> ok; + false -> false + end; + #{scheme := http} -> + ok + end. check_headers(Conf) -> - Method = hocon_schema:get_value("method", Conf), - Headers = hocon_schema:get_value("headers", Conf), + Method = hocon_schema:get_value("config.method", Conf), + Headers = hocon_schema:get_value("config.headers", Conf), case Method =:= get andalso maps:get(<<"content-type">>, Headers, undefined) =/= undefined of true -> false; false -> true From 72c26931d734afdea940907cbb52ed789806055e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 11:01:06 +0200 Subject: [PATCH 2/7] fix: fill string fields' default value with binary --- apps/emqx/src/emqx_authentication.erl | 2 +- apps/emqx/src/emqx_schema.erl | 16 +-- apps/emqx_authn/src/emqx_authn_app.erl | 2 + apps/emqx_authn/src/emqx_authn_schema.erl | 7 ++ .../src/simple_authn/emqx_authn_http.erl | 2 +- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 100 ++++++++++++++++++ apps/emqx_authn/test/emqx_authn_test_lib.erl | 38 +++++++ .../src/emqx_connector_http.erl | 4 +- 8 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 apps/emqx_authn/test/emqx_authn_api_SUITE.erl create mode 100644 apps/emqx_authn/test/emqx_authn_test_lib.erl diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index c96db115f..5ee8b2815 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -251,7 +251,7 @@ authenticator_id(Config) -> emqx_authentication_config:authenticator_id(Config). %% @doc Call this API to initialize authenticators implemented in another APP. --spec initialize_authentication(chain_name(), config()) -> ok. +-spec initialize_authentication(chain_name(), [config()]) -> ok. initialize_authentication(_, []) -> ok; initialize_authentication(ChainName, AuthenticatorsConfig) -> _ = create_chain(ChainName), diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f9b429e16..245d7d5fc 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1042,7 +1042,7 @@ In case PSK cipher suites are intended, make sure to configured , {"ciphers", ciphers_schema(D("ciphers"))} , {user_lookup_fun, sc(typerefl:alias("string", any()), - #{ default => "emqx_tls_psk:lookup" + #{ default => <<"emqx_tls_psk:lookup">> , converter => fun ?MODULE:parse_user_lookup_fun/1 }) } @@ -1191,17 +1191,21 @@ RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA\"
_ -> "" end}). -default_ciphers(undefined) -> - default_ciphers(tls_all_available); -default_ciphers(quic) -> [ +default_ciphers(Which) -> + lists:map(fun erlang:iolist_to_binary/1, + do_default_ciphers(Which)). + +do_default_ciphers(undefined) -> + do_default_ciphers(tls_all_available); +do_default_ciphers(quic) -> [ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256" ]; -default_ciphers(dtls_all_available) -> +do_default_ciphers(dtls_all_available) -> %% as of now, dtls does not support tlsv1.3 ciphers emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']); -default_ciphers(tls_all_available) -> +do_default_ciphers(tls_all_available) -> emqx_tls_lib:default_ciphers(). %% @private return a list of keys in a parent field diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 9284493bf..a59c85e9c 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -25,6 +25,8 @@ , stop/1 ]). +-dialyzer({nowarn_function, [start/2]}). + %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index ca5fe3cc7..c0f16b3f3 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -19,8 +19,15 @@ -include_lib("typerefl/include/types.hrl"). -export([ common_fields/0 + , roots/0 + , fields/1 ]). +%% just a stub, never used at root level +roots() -> []. + +fields(_) -> []. + common_fields() -> [ {enable, fun enable/1} ]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 5ee7a8db8..bbfdaf319 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -102,7 +102,7 @@ body(validator) -> [fun check_body/1]; body(_) -> undefined. request_timeout(type) -> emqx_schema:duration_ms(); -request_timeout(default) -> "5s"; +request_timeout(default) -> <<"5s">>; request_timeout(_) -> undefined. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl new file mode 100644 index 000000000..54c689747 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -0,0 +1,100 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v5"). +-define(BASE_PATH, "api"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_authn, emqx_dashboard], fun set_special_configs/1), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]), + ok. + +set_special_configs(emqx_dashboard) -> + Config = #{ + default_username => <<"admin">>, + default_password => <<"public">>, + listeners => [#{ + protocol => http, + port => 18083 + }] + }, + emqx_config:put([emqx_dashboard], Config), + emqx_config:put([node, data_dir], "data"), + ok; +set_special_configs(_App) -> + ok. + +t_create_http_authn(_) -> + {ok, 200, _} = request(post, uri(["authentication"]), + emqx_authn_test_lib:http_example()), + {ok, 200, _} = request(get, uri(["authentication"])). + +request(Method, Url) -> + request(Method, Url, []). + +request(Method, Url, Body) -> + Request = + case Body of + [] -> + {Url, [auth_header()]}; + _ -> + {Url, [auth_header()], "application/json", to_json(Body)} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +uri() -> uri([]). +uri(Parts) when is_list(Parts) -> + NParts = [E || E <- Parts], + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +get_sources(Result) -> jsx:decode(Result). + +auth_header() -> + Username = <<"admin">>, + Password = <<"public">>, + {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), + {"Authorization", "Bearer " ++ binary_to_list(Token)}. + +to_json(Hocon) -> + {ok, Map} =hocon:binary(Hocon), + jiffy:encode(Map). diff --git a/apps/emqx_authn/test/emqx_authn_test_lib.erl b/apps/emqx_authn/test/emqx_authn_test_lib.erl new file mode 100644 index 000000000..e30854318 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_test_lib.erl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_test_lib). + +-compile(nowarn_export_all). +-compile(export_all). + +http_example() -> +""" +{ + mechanism = \"password-based\" + backend = http + method = post + url = \"http://127.0.0.2:8080\" + headers = {\"content-type\" = \"application/json\"} + body = {username = \"${username}\", + password = \"${password}\"} + pool_size = 8 + connect_timeout = 5000 + request_timeout = 5000 + enable_pipelining = true + ssl = {enable = false} +} +""". diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 7c71e09b3..c724ddb7a 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -93,7 +93,7 @@ base_url(validator) -> fun(#{query := _Query}) -> base_url(_) -> undefined. connect_timeout(type) -> emqx_schema:duration_ms(); -connect_timeout(default) -> "5s"; +connect_timeout(default) -> <<"5s">>; connect_timeout(_) -> undefined. max_retries(type) -> non_neg_integer(); @@ -101,7 +101,7 @@ max_retries(default) -> 5; max_retries(_) -> undefined. retry_interval(type) -> emqx_schema:duration(); -retry_interval(default) -> "1s"; +retry_interval(default) -> <<"1s">>; retry_interval(_) -> undefined. pool_type(type) -> pool_type(); From 88d891a59a1aafcb3b7ae50fcad31ad42e1f0fa3 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 13:04:55 +0200 Subject: [PATCH 3/7] feat: support check_config callback for authenticator provider --- apps/emqx/src/emqx_authentication.erl | 5 ++- apps/emqx/src/emqx_authentication_config.erl | 43 ++++++++++++++----- apps/emqx/test/emqx_authentication_SUITE.erl | 44 ++++++++++++++------ 3 files changed, 67 insertions(+), 25 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 5ee8b2815..c3bfe7912 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -116,7 +116,9 @@ -type user_info() :: #{user_id := binary(), atom() => term()}. --callback refs() -> [{ref, Module, Name}] when Module::module(), Name::atom(). +%% @doc check_config takes raw config from config file, +%% parse and validate it, and reutrn parsed result. +-callback check_config(config()) -> config(). -callback create(Config) -> {ok, State} @@ -176,6 +178,7 @@ , update_user/3 , lookup_user/3 , list_users/1 + , check_config/1 ]). %%------------------------------------------------------------------------------ diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 526d9d6ff..86182143a 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -27,6 +27,11 @@ , authn_type/1 ]). +%% TODO: certs handling should be moved out of emqx app +-ifdef(TEST). +-export([convert_certs/2, convert_certs/3, diff_cert/2, clear_certs/2]). +-endif. + -export_type([config/0]). -include("logger.hrl"). @@ -151,17 +156,33 @@ do_check_conifg(Config, Providers) -> providers => Providers}), throw(unknown_authn_type); Module -> - %% TODO: check if Module:check_config/1 is exported - %% so we do not force all providers to implement hocon schema - try hocon_schema:check_plain(Module, #{<<"config">> => Config}, - #{atom_key => true}) of - #{config := Result} -> - Result - catch - C : E : S -> - ?SLOG(warning, #{msg => "failed_to_check_config", config => Config}), - erlang:raise(C, E, S) - end + do_check_conifg(Type, Config, Module) + end. + +do_check_conifg(Type, Config, Module) -> + F = case erlang:function_exported(Module, check_config, 1) of + true -> + fun Module:check_config/1; + false -> + fun(C) -> + #{config := R} = + hocon_schema:check_plain(Module, #{<<"config">> => C}, + #{atom_key => true}), + R + end + end, + try + F(Config) + catch + C : E : S -> + ?SLOG(warning, #{msg => "failed_to_check_config", + config => Config, + type => Type, + exception => C, + reason => E, + stacktrace => S + }), + throw(bad_authenticator_config) end. return_map([L]) -> L; diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index c4ed85125..cf1514918 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -26,13 +26,13 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("typerefl/include/types.hrl"). --export([ fields/1 ]). +-export([ roots/0, fields/1 ]). --export([ refs/0 - , create/1 +-export([ create/1 , update/2 , authenticate/2 , destroy/1 + , check_config/1 ]). -define(AUTHN, emqx_authentication). @@ -42,6 +42,8 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +roots() -> [{config, #{type => hoconsc:union([hoconsc:ref(type1), hoconsc:ref(type2)])}}]. + fields(type1) -> [ {mechanism, {enum, ['password-based']}} , {backend, {enum, ['built-in-database']}} @@ -62,10 +64,11 @@ enable(_) -> undefined. %% Callbacks %%------------------------------------------------------------------------------ -refs() -> - [ hoconsc:ref(?MODULE, type1) - , hoconsc:ref(?MODULE, type2) - ]. +check_config(C) -> + #{config := R} = + hocon_schema:check_plain(?MODULE, #{<<"config">> => C}, + #{atom_key => true}), + R. create(_Config) -> {ok, #{mark => 1}}. @@ -268,14 +271,14 @@ t_convert_certs(Config) when is_list(Config) -> , {<<"cacertfile">>, "cacert.pem"} ]), - CertsDir = ?AUTHN:certs_dir([Global, <<"password-based:built-in-database">>]), - #{<<"ssl">> := NCerts} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs}), + CertsDir = certs_dir(Config, [Global, <<"password-based:built-in-database">>]), + #{<<"ssl">> := NCerts} = convert_certs(CertsDir, #{<<"ssl">> => Certs}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, Certs))), Certs2 = certs([ {<<"keyfile">>, "key.pem"} , {<<"certfile">>, "cert.pem"} ]), - #{<<"ssl">> := NCerts2} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs2}, #{<<"ssl">> => NCerts}), + #{<<"ssl">> := NCerts2} = convert_certs(CertsDir, #{<<"ssl">> => Certs2}, #{<<"ssl">> => NCerts}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, Certs2))), ?assertEqual(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, NCerts2)), ?assertEqual(maps:get(<<"certfile">>, NCerts), maps:get(<<"certfile">>, NCerts2)), @@ -284,13 +287,13 @@ t_convert_certs(Config) when is_list(Config) -> , {<<"certfile">>, "client-cert.pem"} , {<<"cacertfile">>, "cacert.pem"} ]), - #{<<"ssl">> := NCerts3} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs3}, #{<<"ssl">> => NCerts2}), + #{<<"ssl">> := NCerts3} = convert_certs(CertsDir, #{<<"ssl">> => Certs3}, #{<<"ssl">> => NCerts2}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts3), maps:get(<<"keyfile">>, Certs3))), ?assertNotEqual(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, NCerts3)), ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)), ?assertEqual(true, filelib:is_regular(maps:get(<<"keyfile">>, NCerts3))), - ?AUTHN:clear_certs(CertsDir, #{<<"ssl">> => NCerts3}), + clear_certs(CertsDir, #{<<"ssl">> => NCerts3}), ?assertEqual(false, filelib:is_regular(maps:get(<<"keyfile">>, NCerts3))). update_config(Path, ConfigRequest) -> @@ -305,7 +308,22 @@ certs(Certs) -> diff_cert(CertFile, CertPem2) -> {ok, CertPem1} = file:read_file(CertFile), - ?AUTHN:diff_cert(CertPem1, CertPem2). + emqx_authentication_config:diff_cert(CertPem1, CertPem2). register_provider(Type, Module) -> ok = ?AUTHN:register_providers([{Type, Module}]). + +certs_dir(CtConfig, Path) -> + DataDir = proplists:get_value(data_dir, CtConfig), + Dir = filename:join([DataDir | Path]), + filelib:ensure_dir(Dir), + Dir. + +convert_certs(CertsDir, SslConfig) -> + emqx_authentication_config:convert_certs(CertsDir, SslConfig). + +convert_certs(CertsDir, New, Old) -> + emqx_authentication_config:convert_certs(CertsDir, New, Old). + +clear_certs(CertsDir, SslConfig) -> + emqx_authentication_config:clear_certs(CertsDir, SslConfig). From 019b9d17f6499d6023812a15188a19e5161d9842 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 13:13:03 +0200 Subject: [PATCH 4/7] refactor(emqx_config_handler): async remove --- apps/emqx/src/emqx_authentication.erl | 10 ++++++++-- apps/emqx/src/emqx_authentication_config.erl | 12 ++++++------ apps/emqx/src/emqx_config_handler.erl | 11 +++++------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index c3bfe7912..793a96cf2 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -557,8 +557,14 @@ handle_info(Info, State) -> {noreply, State}. terminate(Reason, _State) -> - ?SLOG(error, #{msg => "emqx_authentication_terminating", - reason => Reason}), + case Reason of + normal -> ok; + {shutdown, _} -> ok; + Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating", + reason => Other}) + end, + emqx_config_handler:remove_handler([authentication]), + emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 86182143a..2bea0cbe9 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -221,12 +221,12 @@ convert_certs(CertsDir, NewConfig, OldConfig) -> OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), Diff = diff_certs(NewSSLOpts, OldSSLOpts), NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> - Acc#{K => maps:get(K, OldSSLOpts)}; - ({_, K}, Acc) -> - CertFile = generate_filename(CertsDir, K), - ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), - Acc#{K => CertFile} - end, NewSSLOpts, Diff), + Acc#{K => maps:get(K, OldSSLOpts)}; + ({_, K}, Acc) -> + CertFile = generate_filename(CertsDir, K), + ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), + Acc#{K => CertFile} + end, NewSSLOpts, Diff), NewConfig#{<<"ssl">> => NSSLOpts} end. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 297f1147d..c44a0cb96 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -77,9 +77,10 @@ update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> add_handler(ConfKeyPath, HandlerName) -> gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}, infinity). +%% @doc Remove handler asynchronously -spec remove_handler(emqx_config:config_key_path()) -> ok. remove_handler(ConfKeyPath) -> - gen_server:call(?MODULE, {remove_handler, ConfKeyPath}, infinity). + gen_server:cast(?MODULE, {remove_handler, ConfKeyPath}). %%============================================================================ @@ -95,11 +96,6 @@ handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers : {reply, Error, State} end; -handle_call({remove_handler, ConfKeyPath}, _From, - State = #{handlers := Handlers}) -> - {reply, ok, State#{handlers => - emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}}; - handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, #{handlers := Handlers} = State) -> Reply = try @@ -125,6 +121,9 @@ handle_call(_Request, _From, State) -> Reply = ok, {reply, Reply, State}. +handle_cast({remove_handler, ConfKeyPath}, + State = #{handlers := Handlers}) -> + {noreply, State#{handlers => emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}}; handle_cast(_Msg, State) -> {noreply, State}. From d468c21e61c5d399b174d7c51d964f97318d5d0c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 14:36:37 +0200 Subject: [PATCH 5/7] fix(authn): rename config key http-server to http --- apps/emqx_authn/src/emqx_authn.erl | 3 +-- apps/emqx_authn/src/emqx_authn_api.erl | 8 ++++---- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 1b0d12fd2..a49aacff4 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -28,8 +28,7 @@ providers() -> , {{'password-based', postgresql}, emqx_authn_pgsql} , {{'password-based', mongodb}, emqx_authn_mongodb} , {{'password-based', redis}, emqx_authn_redis} - , {{'password-based', 'http-server'}, emqx_authn_http} - , {{'password-based', 'http'}, emqx_authn_http} %% TODO: resolve one + , {{'password-based', 'http'}, emqx_authn_http} , {jwt, emqx_authn_jwt} , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} ]. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 93df83829..f8e8bf5de 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -43,7 +43,7 @@ }}). -define(EXAMPLE_2, #{mechanism => <<"password-based">>, - backend => <<"http-server">>, + backend => <<"http">>, method => <<"post">>, url => <<"http://localhost:80/login">>, headers => #{ @@ -90,7 +90,7 @@ -define(INSTANCE_EXAMPLE_1, maps:merge(?EXAMPLE_1, #{id => <<"password-based:built-in-database">>, enable => true})). --define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http-server">>, +-define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http">>, connect_timeout => "5s", enable_pipelining => true, headers => #{ @@ -1506,8 +1506,8 @@ definitions() -> }, backend => #{ type => string, - enum => [<<"http-server">>], - example => <<"http-server">> + enum => [<<"http">>], + example => <<"http">> }, method => #{ type => string, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index bbfdaf319..c99806341 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -61,8 +61,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['http-server', 'http']}} %% TODO: delete http + [ {mechanism, 'password-based'} + , {backend, 'http'} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} From fa91e5f583b7acb7656c1e5864948fdccd6f4f71 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 15:06:08 +0200 Subject: [PATCH 6/7] fix(authn): allow single authenticator instance from config --- apps/emqx/src/emqx_schema.erl | 19 +++++++++++++------ apps/emqx_authn/src/emqx_authn.erl | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 245d7d5fc..4b1f0c55f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -103,12 +103,10 @@ The configs here work as default values which can be overriden in zone configs""" })} , {"authentication", - sc(hoconsc:lazy(hoconsc:array(map())), - #{ desc => + authentication( """Default authentication configs for all MQTT listeners.
For per-listener overrides see authentication -in listener configs""" - })} +in listener configs""")} , {"authorization", sc(ref("authorization"), #{})} @@ -903,8 +901,7 @@ mqtt_listener() -> #{}) } , {"authentication", - sc(hoconsc:lazy(hoconsc:array(map())), - #{}) + authentication("Per-listener authentication override") } ]. @@ -1356,3 +1353,13 @@ str(B) when is_binary(B) -> binary_to_list(B); str(S) when is_list(S) -> S. + +authentication(Desc) -> + #{ type => hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())]) + , desc => [Desc, "
", """ +Authentication can be one single authenticator instance or a chain of authenticators as an array. +The when authenticating a login (username, client ID, etc.) the authenticators are checked +in the configured order. +""" + ] + }. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index a49aacff4..fbd31c5d2 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -33,6 +33,8 @@ providers() -> , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} ]. +check_configs(C) when is_map(C) -> + check_configs([C]); check_configs([]) -> []; check_configs([Config | Configs]) -> [check_config(Config) | check_configs(Configs)]. From ce5b45687252a892daf80cfc19173c9e5e8c8e26 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Oct 2021 17:15:16 +0200 Subject: [PATCH 7/7] refactor(authn): make schema doc generation work --- apps/emqx/src/emqx_schema.erl | 11 ++++++----- apps/emqx_authn/src/emqx_authn_schema.erl | 9 +++++++-- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 11 +++++------ apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl | 7 +++---- .../emqx_authn/src/simple_authn/emqx_authn_mnesia.erl | 2 +- .../src/simple_authn/emqx_authn_mongodb.erl | 8 +++----- apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl | 2 +- apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl | 2 +- apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl | 8 +++----- apps/emqx_machine/src/emqx_machine_schema.erl | 1 + 10 files changed, 31 insertions(+), 30 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 4b1f0c55f..5958167cd 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1355,11 +1355,12 @@ str(S) when is_list(S) -> S. authentication(Desc) -> - #{ type => hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())]) - , desc => [Desc, "
", """ + #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])) + , desc => iolist_to_binary([Desc, "
", """ Authentication can be one single authenticator instance or a chain of authenticators as an array. The when authenticating a login (username, client ID, etc.) the authenticators are checked -in the configured order. -""" - ] +in the configured order.
+EMQ X comes with a set of pre-built autenticators, for more details, see +authenticator_config. +"""]) }. diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index c0f16b3f3..b36e88ebf 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -23,8 +23,10 @@ , fields/1 ]). -%% just a stub, never used at root level -roots() -> []. +%% only for doc generation +roots() -> [{authenticator_config, + #{type => hoconsc:union(config_refs([Module || {_AuthnType, Module} <- emqx_authn:providers()])) + }}]. fields(_) -> []. @@ -35,3 +37,6 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; enable(_) -> undefined. + +config_refs(Modules) -> + lists:append([Module:refs() || Module <- Modules]). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index c99806341..ceb4b30a8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -40,12 +40,11 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-http_server". +namespace() -> "authn-http". roots() -> - [ {config, {union, [ hoconsc:ref(?MODULE, get) - , hoconsc:ref(?MODULE, post) - ]}} + [ {config, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(get) -> @@ -61,8 +60,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, 'password-based'} - , {backend, 'http'} + [ {mechanism, hoconsc:enum(['password-based'])} + , {backend, hoconsc:enum(['http'])} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 1b7c5b87d..c4e04eac3 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -40,10 +40,9 @@ namespace() -> "authn-jwt". roots() -> - [ {config, {union, [ hoconsc:mk('hmac-based') - , hoconsc:mk('public-key') - , hoconsc:mk('jwks') - ]}} + [ {config, hoconsc:mk(hoconsc:union(refs()), + #{} + )} ]. fields('hmac-based') -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index c3b6badf7..2df09c043 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -84,7 +84,7 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-builtin_db". +namespace() -> "authn-builtin_db". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 7b986e6f9..ce5d3d8ee 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -39,13 +39,11 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-mongodb". +namespace() -> "authn-mongodb". roots() -> - [ {config, {union, [ hoconsc:mk(standalone) - , hoconsc:mk('replica-set') - , hoconsc:mk('sharded-cluster') - ]}} + [ {config, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index b4c6dac08..fb090aca2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -39,7 +39,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-mysql". +namespace() -> "authn-mysql". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 5f1005e9f..4086f4b22 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -40,7 +40,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-postgresql". +namespace() -> "authn-postgresql". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index cb04b0274..3ae333d12 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -39,13 +39,11 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-password_based-redis". +namespace() -> "authn-redis". roots() -> - [ {config, {union, [ hoconsc:mk(standalone) - , hoconsc:mk(cluster) - , hoconsc:mk(sentinel) - ]}} + [ {config, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 995de1f62..ce6fc74db 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -45,6 +45,7 @@ [ emqx_bridge_schema , emqx_retainer_schema , emqx_statsd_schema + , emqx_authn_schema , emqx_authz_schema , emqx_auto_subscribe_schema , emqx_modules_schema