From 5d3cb6ae1cc937808cbb61fd1cb9f485613d7447 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 29 Nov 2021 14:51:49 +0100 Subject: [PATCH 01/16] refactor(authn): align authn config root name authn configs are checked independently per-auth provider, this was to make authn providers more plugable. in order to make environment variable overrides work for authn, we need to have a unified view of the config layout, no matter from root level, or partially checking per-provider config independently, i.e. we try to use the same config envelop. --- apps/emqx/include/emqx_authentication.hrl | 31 +++++++++++++++++ apps/emqx/src/emqx_authentication.erl | 11 +++--- apps/emqx/src/emqx_authentication_config.erl | 20 ++++++----- apps/emqx/src/emqx_config.erl | 7 ++-- apps/emqx/src/emqx_schema.erl | 22 +++++++++--- apps/emqx/test/emqx_authentication_SUITE.erl | 27 ++++++--------- apps/emqx_authn/etc/emqx_authn.conf | 7 +--- apps/emqx_authn/include/emqx_authn.hrl | 7 ++++ apps/emqx_authn/src/emqx_authn.erl | 17 +++++++--- apps/emqx_authn/src/emqx_authn_api.erl | 7 ++-- apps/emqx_authn/src/emqx_authn_app.erl | 6 ++-- apps/emqx_authn/src/emqx_authn_schema.erl | 22 ++++++++++-- .../emqx_enhanced_authn_scram_mnesia.erl | 10 +++--- .../src/simple_authn/emqx_authn_http.erl | 20 ++++++----- .../src/simple_authn/emqx_authn_jwt.erl | 9 ++--- .../src/simple_authn/emqx_authn_mnesia.erl | 12 +++---- .../src/simple_authn/emqx_authn_mongodb.erl | 8 ++--- .../src/simple_authn/emqx_authn_mysql.erl | 10 +++--- .../src/simple_authn/emqx_authn_pgsql.erl | 10 +++--- .../src/simple_authn/emqx_authn_redis.erl | 14 ++++---- .../test/emqx_authn_mnesia_SUITE.erl | 6 ++-- apps/emqx_conf/src/emqx_conf_schema.erl | 6 ++++ .../src/coap/emqx_coap_channel.erl | 11 +++--- apps/emqx_gateway/src/emqx_gateway_api.erl | 5 +-- apps/emqx_gateway/src/emqx_gateway_conf.erl | 34 ++++++++++--------- apps/emqx_gateway/src/emqx_gateway_http.erl | 7 ++-- apps/emqx_gateway/src/emqx_gateway_schema.erl | 33 +++++------------- 27 files changed, 231 insertions(+), 148 deletions(-) create mode 100644 apps/emqx/include/emqx_authentication.hrl diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl new file mode 100644 index 000000000..948842433 --- /dev/null +++ b/apps/emqx/include/emqx_authentication.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_AUTHENTICATION_HRL). +-define(EMQX_AUTHENTICATION_HRL, true). + +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + +%% key to a persistent term which stores a module name in order to inject +%% schema module at run-time to keep emqx app's compile time purity. +%% see emqx_schema.erl for more details +%% and emqx_conf_schema for an examples +-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module). + +-endif. diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 77a5e2cee..1607497da 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -24,9 +24,12 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + %% The authentication entrypoint. -export([ authenticate/2 ]). @@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) -> %%-------------------------------------------------------------------- init(_Opts) -> - ok = emqx_config_handler:add_handler([authentication], ?MODULE), - ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE), {ok, #{hooked => false, providers => #{}}}. handle_call(get_providers, _From, #{providers := Providers} = State) -> @@ -496,8 +499,8 @@ terminate(Reason, _State) -> Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating", reason => Other}) end, - emqx_config_handler:remove_handler([authentication]), - emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + emqx_config_handler:remove_handler([?CONF_ROOT]), + emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index a7fa5673a..795dd060e 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -34,6 +34,7 @@ -export_type([config/0]). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -type parsed_config() :: #{mechanism := atom(), backend => atom(), @@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} check_configs(Configs) -> Providers = emqx_authentication:get_providers(), - lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs). + lists:map(fun(C) -> do_check_config(C, Providers) end, Configs). -do_check_conifg(Config, Providers) -> +do_check_config(Config, Providers) -> Type = authn_type(Config), case maps:get(Type, Providers, false) of false -> @@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) -> providers => Providers}), throw({unknown_authn_type, Type}); Module -> - do_check_conifg(Type, Config, Module) + do_check_config(Type, Config, Module) end. -do_check_conifg(Type, Config, Module) -> +do_check_config(Type, Config, Module) -> F = case erlang:function_exported(Module, check_config, 1) of true -> fun Module:check_config/1; false -> fun(C) -> - #{config := R} = - hocon_schema:check_plain(Module, #{<<"config">> => C}, + Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + R = hocon_schema:check_plain(Module, #{Key => C}, #{atom_key => true}), - R + maps:get(AtomKey, R) end end, try @@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M); authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)}; authn_type(#{<<"mechanism">> := M}) -> atom(M). -atom(Bin) -> - binary_to_existing_atom(Bin, utf8). +atom(A) when is_atom(A) -> A; +atom(Bin) -> binary_to_existing_atom(Bin, utf8). %% The relative dir for ssl files. certs_dir(ChainName, ConfigOrID) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 8806d4bc3..2693d559e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -268,12 +268,13 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> +init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), %% check and save configs - {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0), + {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(get_root_names(), RawConf0)). + maps:with(RootNames, RawConf)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e470ff478..c137f4a37 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -22,6 +22,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include("emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type duration() :: integer(). @@ -105,7 +106,7 @@ and can not be deleted.""" The configs here work as default values which can be overriden in zone configs""" })} - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication( """Default authentication configs for all MQTT listeners.
For per-listener overrides see authentication @@ -972,7 +973,7 @@ mqtt_listener() -> sc(duration(), #{}) } - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication("Per-listener authentication override") } ]. @@ -1436,8 +1437,21 @@ str(S) when is_list(S) -> S. authentication(Desc) -> - #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])) - , desc => iolist_to_binary([Desc, "
", """ + %% authentication schemais lazy to make it more 'plugable' + %% the type checks are done in emqx_auth application when it boots. + %% and in emqx_authentication_config module for rutime changes. + Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])), + %% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY + %% is for now only affecting document generation. + %% maybe in the future, we can find a more straightforward way to support + %% * document generation (at compile time) + %% * type checks before boot (in bin/emqx config generation) + %% * type checks at runtime (when changing configs via management API) + #{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of + undefined -> Default; + Module -> hoconsc:lazy(Module:root_type()) + end + , desc => iolist_to_binary([Desc, """ Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 10a4e4091..2c28a6076 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -25,18 +25,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("typerefl/include/types.hrl"). - --export([ roots/0, fields/1 ]). - --export([ create/2 - , update/2 - , authenticate/2 - , destroy/1 - , check_config/1 - ]). +-include("emqx_authentication.hrl"). -define(AUTHN, emqx_authentication). -define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -250,7 +243,7 @@ t_update_config({init, Config}) -> {"auth2", AuthNType2} | Config]; t_update_config(Config) when is_list(Config) -> - emqx_config_handler:add_handler([authentication], emqx_authentication), + emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication), ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), @@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})), ?assertMatch( {ok, #{id := ID1, state := #{mark := 1}}}, @@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch( {ok, #{id := ID2, state := #{mark := 1}}}, @@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], + update_config([?CONF_ROOT], {update_authenticator, Global, ID1, @@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {move_authenticator, Global, ID2, top})), + update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})), ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), - ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})), ?assertEqual( {error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch( {ok, _}, - update_config([authentication], {delete_authenticator, Global, ID2})), + update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})), ?assertEqual( {error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)), ListenerID = 'tcp:default', - ConfKeyPath = [listeners, tcp, default, authentication], + ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT], ?assertMatch( {ok, _}, diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index d1d3d16f8..7503f3b89 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,6 +1 @@ -# authentication: { -# mechanism: password-based -# backend: built-in-database -# user_id_type: clientid -# } - +authentication: [] diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 1c5c00e85..b5dfa8f01 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -17,6 +17,8 @@ -ifndef(EMQX_AUTHN_HRL). -define(EMQX_AUTHN_HRL, true). +-include_lib("emqx/include/emqx_authentication.hrl"). + -define(APP, emqx_authn). -define(AUTHN, emqx_authentication). @@ -27,4 +29,9 @@ -define(AUTH_SHARD, emqx_authn_shard). +%% has to be the same as the root field name defined in emqx_schema +-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME). +-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). +-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -endif. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index fbd31c5d2..5b3d76822 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -22,6 +22,8 @@ , check_configs/1 ]). +-include("emqx_authn.hrl"). + providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} @@ -44,8 +46,8 @@ check_config(Config) -> check_config(Config, Opts) -> case do_check_config(Config, Opts) of - #{config := Checked} -> Checked; - #{<<"config">> := WithDefaults} -> WithDefaults + #{?CONF_NS_ATOM := Checked} -> Checked; + #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> @@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> case lists:keyfind(Key, 1, providers()) of false -> throw({unknown_handler, Key}); - {_, Provider} -> - hocon_schema:check_plain(Provider, #{<<"config">> => Config}, + {_, ProviderModule} -> + hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config}, Opts#{atom_key => true}) end. atom(Bin) -> - binary_to_existing_atom(Bin, utf8). + try + binary_to_existing_atom(Bin, utf8) + catch + _ : _ -> + throw({unknown_auth_provider, Bin}) + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 83ca9e56d..81964f3ea 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,6 +22,7 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -import(hoconsc, [mk/2, ref/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -32,8 +33,10 @@ % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]). --define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]). +-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">>]). +-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">>]). -export([ api_spec/0 , paths/0 diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index df7fbecd3..035e61910 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -25,6 +25,8 @@ , stop/1 ]). +-include_lib("emqx/include/emqx_authentication.hrl"). + -dialyzer({nowarn_function, [start/2]}). %%------------------------------------------------------------------------------ @@ -65,7 +67,7 @@ chain_configs() -> [global_chain_config() | listener_chain_configs()]. global_chain_config() -> - {?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}. + {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}. listener_chain_configs() -> lists:map( @@ -77,7 +79,7 @@ listener_chain_configs() -> auth_config_path(ListenerID) -> [<<"listeners">>] ++ binary:split(atom_to_binary(ListenerID), <<":">>) - ++ [<<"authentication">>]. + ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY]. provider_types() -> lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 22f62f519..c2e963ec4 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,10 +22,12 @@ , roots/0 , fields/1 , authenticator_type/0 + , root_type/0 + , mechanism/1 + , backend/1 ]). -%% only for doc generation -roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}]. +roots() -> []. fields(_) -> []. @@ -35,6 +37,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(desc) -> "Set to false to disable this auth provider"; enable(_) -> undefined. authenticator_type() -> @@ -42,3 +45,18 @@ authenticator_type() -> config_refs(Modules) -> lists:append([Module:refs() || Module <- Modules]). + +%% authn is a core functionality however implemented outside fo emqx app +%% in emqx_schema, 'authentication' is a map() type which is to allow +%% EMQ X more plugable. +root_type() -> + T = authenticator_type(), + hoconsc:union([T, hoconsc:array(T)]). + +mechanism(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). + +backend(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index f02655462..3604455dc 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -83,11 +83,11 @@ mnesia(boot) -> namespace() -> "authn-scram-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, [scram]}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('scram')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). @@ -105,7 +105,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{algorithm := Algorithm, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 829ce2282..533301ea7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -43,8 +43,9 @@ namespace() -> "authn-http". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(get) -> @@ -60,8 +61,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, hoconsc:enum(['password-based'])} - , {backend, hoconsc:enum(['http'])} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(http)} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} @@ -233,9 +234,9 @@ transform_header_name(Headers) -> end, #{}, Headers). check_ssl_opts(Conf) -> - case parse_url(hocon_schema:get_value("config.url", Conf)) of + case parse_url(get_conf_val("url", Conf)) of #{scheme := https} -> - case hocon_schema:get_value("config.ssl.enable", Conf) of + case get_conf_val("ssl.enable", Conf) of true -> ok; false -> false end; @@ -244,8 +245,8 @@ check_ssl_opts(Conf) -> end. check_headers(Conf) -> - Method = to_bin(hocon_schema:get_value("config.method", Conf)), - Headers = hocon_schema:get_value("config.headers", Conf), + Method = to_bin(get_conf_val("method", Conf)), + Headers = get_conf_val("headers", Conf), Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). parse_url(URL) -> @@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L). + +get_conf_val(Name, Conf) -> + hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 916911ffa..9295e5c7e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -16,6 +16,7 @@ -module(emqx_authn_jwt). +-include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -40,9 +41,9 @@ namespace() -> "authn-jwt". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{} - )} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields('hmac-based') -> @@ -82,7 +83,7 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {mechanism, {enum, [jwt]}} + [ {mechanism, emqx_authn_schema:mechanism('jwt')} , {verify_claims, fun verify_claims/1} ] ++ emqx_authn_schema:common_fields(). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index fd02671fb..f609d8cac 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -85,11 +85,11 @@ mnesia(boot) -> namespace() -> "authn-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} ] ++ emqx_authn_schema:common_fields(); @@ -104,7 +104,7 @@ fields(other_algorithms) -> ]. user_id_type(type) -> user_id_type(); -user_id_type(default) -> username; +user_id_type(default) -> <<"username">>; user_id_type(_) -> undefined. password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), @@ -121,7 +121,7 @@ salt_rounds(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{user_id_type := Type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 2deef8506..3b47bcd7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -42,8 +42,8 @@ namespace() -> "authn-mongodb". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,8 +56,8 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mongodb]}} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mongodb)} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 47ca0ae3c..fd0d09f57 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -41,11 +41,11 @@ namespace() -> "authn-mysql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mysql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mysql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -74,7 +74,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 660acf566..fdd30b618 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -47,11 +47,11 @@ namespace() -> "authn-postgresql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [postgresql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(postgresql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -75,7 +75,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index d238bc537..e17d0ad8f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -42,8 +42,8 @@ namespace() -> "authn-redis". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,11 +56,11 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [{mechanism, {enum, ['password-based']}}, - {backend, {enum, [redis]}}, - {cmd, fun cmd/1}, - {password_hash_algorithm, fun password_hash_algorithm/1}, - {salt_position, fun salt_position/1} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(redis)} + , {cmd, fun cmd/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} ] ++ emqx_authn_schema:common_fields(). cmd(type) -> string(); diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index b9eadbef6..a9e474f11 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -47,6 +47,8 @@ end_per_testcase(_Case, Config) -> %% Tests %%------------------------------------------------------------------------------ +-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}). + t_check_schema(_Config) -> ConfigOk = #{ <<"mechanism">> => <<"password-based">>, @@ -58,7 +60,7 @@ t_check_schema(_Config) -> } }, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}), + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)), ConfigNotOk = #{ <<"mechanism">> => <<"password-based">>, @@ -72,7 +74,7 @@ t_check_schema(_Config) -> ?assertException( throw, {emqx_authn_mnesia, _}, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})). + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))). t_create(_) -> Config0 = config(), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 617703cd2..a84418916 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -24,6 +24,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -62,6 +63,11 @@ namespace() -> undefined. roots() -> + PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, + case persistent_term:get(PtKey, undefined) of + undefined -> persistent_term:put(PtKey, emqx_authn_schema); + _ -> ok + end, %% authorization configs are merged in THIS schema's "authorization" fields lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++ [ {"node", diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index d7b0a00c3..e1dad1dee 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -18,9 +18,6 @@ -behaviour(emqx_gateway_channel). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). - %% API -export([ info/1 , info/2 @@ -44,6 +41,12 @@ -export_type([channel/0]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + -record(channel, { %% Context ctx :: emqx_gateway_ctx:context(), @@ -283,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) -> %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - case emqx_conf:get([gateway, coap, authentication], undefined) of + case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of undefined -> call_session(handle_request, Msg, Channel); _ -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index d79a880a1..925b7cc0e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -17,6 +17,7 @@ -module(emqx_gateway_api). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -behaviour(minirest_api). @@ -243,7 +244,7 @@ schema_gateway_overview_list() -> %% %% NOTE: It is a temporary measure to generate swagger-schema -define(COAP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, @@ -331,7 +332,7 @@ schema_gateway_overview_list() -> ). -define(STOMP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 3612b428d..e97b0062d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -17,8 +17,6 @@ %% @doc The gateway configuration management module -module(emqx_gateway_conf). --include_lib("emqx/include/logger.hrl"). - %% Load/Unload -export([ load/0 , unload/0 @@ -56,6 +54,10 @@ , post_config_update/5 ]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -type atom_or_bin() :: atom() | binary(). -type ok_or_err() :: ok_or_err(). -type listener_ref() :: {ListenerType :: atom_or_bin(), @@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) -> -spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). update_gateway(GwName, Conf0) -> - Conf = maps:without([listeners, authentication, - <<"listeners">>, <<"authentication">>], Conf0), + Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], + Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0], + Conf = maps:without(Exclude0 ++ Exclude1, Conf0), update({?FUNCTION_NAME, bin(GwName), Conf}). %% FIXME: delete cert files ?? @@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> undefined -> {error, not_found}; _ -> - NConf = maps:without([<<"listeners">>, - <<"authentication">>], Conf), + NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> @@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})}; + #{GwName => #{?AUTHN_BIN => Conf}})}; _ -> {error, already_exist} end; @@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(<<"authentication">>, Conf, Listener), + NListener = maps:put(?AUTHN_BIN, Conf, Listener), NGateway = #{GwName => #{<<"listeners">> => #{LType => #{LName => NListener}}}}, @@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {error, not_found}; _ -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})} + #{GwName => #{?AUTHN_BIN => Conf}})} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case emqx_map_lib:deep_get( @@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> {error, not_found}; Auth -> NListener = maps:put( - <<"authentication">>, + ?AUTHN_BIN, emqx_map_lib:deep_merge(Auth, Conf), Listener ), @@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, emqx_map_lib:deep_remove( - [GwName, <<"authentication">>], RawConf)}; + [GwName, ?AUTHN_BIN], RawConf)}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> - Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>], + Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], {ok, emqx_map_lib:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 0d2f765c5..98344f968 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -19,6 +19,9 @@ -include("include/emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %% Mgmt APIs - gateway -export([ gateways/1 @@ -166,7 +169,7 @@ remove_listener(ListenerId) -> -spec authn(gateway_name()) -> map(). authn(GwName) -> %% XXX: Need append chain-nanme, authenticator-id? - Path = [gateway, GwName, authentication], + Path = [gateway, GwName, ?AUTHN], ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, @@ -176,7 +179,7 @@ authn(GwName) -> -spec authn(gateway_name(), binary()) -> map(). authn(GwName, ListenerId) -> {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), - Path = [gateway, GwName, listeners, Type, Name, authentication], + Path = [gateway, GwName, listeners, Type, Name, ?AUTHN], ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9f35225b7..5f87131d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -24,6 +24,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type ip_port() :: tuple(). @@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW" , desc => "The Pre-defined topic ids and topic names.
A 'pre-defined' topic id is a topic id whose mapping to a topic name -is known in advance by both the client’s application and the gateway" +is known in advance by both the client's application and the gateway" })} , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); @@ -407,30 +408,14 @@ fields(dtls_opts) -> , ciphers => dtls_all_available }, false). -authentication() -> - sc(hoconsc:union( - [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]), - #{ nullable => {true, recursively} - , desc => +authentication_schema() -> + sc(emqx_authn_schema:authenticator_type(), + #{ nullable => {true, recursively} + , desc => """Default authentication configs for all of the gateway listeners.
For per-listener overrides see authentication in listener configs""" - }). + }). gateway_common_options() -> [ {enable, @@ -464,7 +449,7 @@ it has two purposes: sc(ref(clientinfo_override), #{ desc => "" })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} ]. common_listener_opts() -> @@ -483,7 +468,7 @@ common_listener_opts() -> sc(integer(), #{ default => 1000 })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} , {mountpoint, sc(binary(), #{ default => undefined From 867cc3c4ade7c888a9a6ba6481f41a3b084145c3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Dec 2021 19:44:59 +0100 Subject: [PATCH 02/16] test: ensure emqx_connector is loaded --- apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 49056ed68..2bca1793d 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -31,7 +31,7 @@ groups() -> init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], + [emqx_connector, emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. From bfc36efa0eaf72709f2d63446ea91e4f36f3b05f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Dec 2021 20:16:46 +0100 Subject: [PATCH 03/16] fix: pin hocon 0.21.1 --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 10 ++++++++-- build | 2 +- rebar.config | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 59c5cf045..5adcbd3bb 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,7 +17,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 2693d559e..ecb843ed5 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -272,9 +272,15 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), %% check and save configs {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + %% fill default values for raw config + Opts = #{only_fill_defaults => true, + logger => fun(_, _) -> ok end, %% everything should have been logged already + nullable => true %% TODO: evil, remove, nullable should be declared in schema + }, + RawConfWithDefaults = hocon_schema:check_plain(SchemaMod, RawConf, Opts), RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(RootNames, RawConf)). + maps:with(RootNames, RawConfWithDefaults)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. @@ -283,7 +289,7 @@ include_dirs() -> when AppEnvs :: app_envs(), CheckedConf :: config(). check_config(SchemaMod, RawConf) -> Opts = #{return_plain => true, - nullable => true, + nullable => true, %% TODO: evil, remove, nullable should be declared in schema format => map }, {AppEnvs, CheckedConf} = diff --git a/build b/build index c79dc711c..9f9879df4 100755 --- a/build +++ b/build @@ -60,7 +60,7 @@ docgen() { conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md" echo "===< Generating config document $conf_doc_markdown" # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema)), halt(0)." + erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema, \"EMQ X ${PKG_VSN} Configuration\")), halt(0)." } make_rel() { diff --git a/rebar.config b/rebar.config index 4fbf1b136..33c1244d2 100644 --- a/rebar.config +++ b/rebar.config @@ -64,7 +64,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From e6e28f900250ae1d2b6bccd9dc67b76990bb4c3f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 29 Nov 2021 23:57:42 +0100 Subject: [PATCH 04/16] docs: fix emqx_schema doc style --- apps/emqx/src/emqx_schema.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c137f4a37..2415f051c 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1232,16 +1232,18 @@ ciphers_schema(Default) -> false -> fun validate_ciphers/1 end , desc => -"""TLS cipher suite names separated by comma, or as an array of strings +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. \"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or -[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"][\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"].
Ciphers (and their ordering) define the way in which the -client and server encrypts information over the wire. +client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the application's data security, confidentiality and performance. -The names should be in OpenSSL sting format (not RFC format). -Default values and examples proveded by EMQ X config + +The names should be in OpenSSL string format (not RFC format). +All default values and examples proveded by EMQ X config documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with @@ -1455,7 +1457,5 @@ authentication(Desc) -> Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
-EMQ X comes with a set of pre-built autenticators, for more details, see -autenticator_config """]) }. From d9a980c7e1945072ec7afac0ab08d4f7843e00cf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 30 Nov 2021 17:09:13 +0100 Subject: [PATCH 05/16] docs: fix config doc, avoid using in doc body --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_schema.erl | 3 ++- apps/emqx_connector/src/emqx_connector_api.erl | 6 +++--- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 417fa49f7..6e274903e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -33,7 +33,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 26a1d5bd1..3a3151ef0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -108,7 +108,8 @@ connector_name() -> #{ nullable => false , desc =>""" The connector name to be used for this bridge. -Connectors are configured by 'connectors.. +Connectors are configured as 'connectors.type.name', +for example 'connectors.http.mybridge'. """ })}. diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 6eb397519..7e934b997 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -38,7 +38,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). namespace() -> "connector". @@ -74,7 +74,7 @@ schema("/connectors_test") -> post => #{ tags => [<<"connectors">>], description => <<"Test creating a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Test creating connector">>, requestBody => connector_test_info(), responses => #{ @@ -98,7 +98,7 @@ schema("/connectors") -> post => #{ tags => [<<"connectors">>], description => <<"Create a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Create connector">>, requestBody => connector_info(), responses => #{ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 8daae99b5..995044fc7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -56,7 +56,7 @@ A list of outputs of the rule.
An output can be a string that refers to the channel Id of a emqx bridge, or a object that refers to a function.
There a some built-in functions like \"republish\" and \"console\", and we also support user -provided functions like \":\".
+provided functions like \"ModuleName:FunctionName\".
The outputs in the list is executed one by one in order. This means that if one of the output is executing slowly, all of the outputs comes after it will not be executed until it returns.
From 2a3f58f8ed8b688a0b0693e97d7a00326404a43c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 3 Dec 2021 20:23:39 +0100 Subject: [PATCH 06/16] chore: add a comment in bin/emqx to warn the config overlay --- bin/emqx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/emqx b/bin/emqx index e1b72d1f9..920cd00f4 100755 --- a/bin/emqx +++ b/bin/emqx @@ -342,6 +342,9 @@ generate_config() { NOW_TIME="$(call_hocon now_time)" ## ths command populates two files: app.