diff --git a/Makefile b/Makefile index e381db3ed..52e4f3ffc 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: +$1-ct: conf-segs $(REBAR) ct --name $(CT_NODE_NAME) -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl new file mode 100644 index 000000000..948842433 --- /dev/null +++ b/apps/emqx/include/emqx_authentication.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_AUTHENTICATION_HRL). +-define(EMQX_AUTHENTICATION_HRL, true). + +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + +%% key to a persistent term which stores a module name in order to inject +%% schema module at run-time to keep emqx app's compile time purity. +%% see emqx_schema.erl for more details +%% and emqx_conf_schema for an examples +-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module). + +-endif. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 59c5cf045..5adcbd3bb 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,7 +17,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 914651535..35b3767e0 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -63,4 +63,5 @@ do_authorize(ClientInfo, PubSub, Topic) -> -compile({inline, [run_hooks/3]}). run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). + ok = emqx_metrics:inc(Name), + emqx_hooks:run_fold(Name, Args, Acc). diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 77a5e2cee..1607497da 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -24,9 +24,12 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + %% The authentication entrypoint. -export([ authenticate/2 ]). @@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) -> %%-------------------------------------------------------------------- init(_Opts) -> - ok = emqx_config_handler:add_handler([authentication], ?MODULE), - ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE), {ok, #{hooked => false, providers => #{}}}. handle_call(get_providers, _From, #{providers := Providers} = State) -> @@ -496,8 +499,8 @@ terminate(Reason, _State) -> Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating", reason => Other}) end, - emqx_config_handler:remove_handler([authentication]), - emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + emqx_config_handler:remove_handler([?CONF_ROOT]), + emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index a7fa5673a..795dd060e 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -34,6 +34,7 @@ -export_type([config/0]). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -type parsed_config() :: #{mechanism := atom(), backend => atom(), @@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} check_configs(Configs) -> Providers = emqx_authentication:get_providers(), - lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs). + lists:map(fun(C) -> do_check_config(C, Providers) end, Configs). -do_check_conifg(Config, Providers) -> +do_check_config(Config, Providers) -> Type = authn_type(Config), case maps:get(Type, Providers, false) of false -> @@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) -> providers => Providers}), throw({unknown_authn_type, Type}); Module -> - do_check_conifg(Type, Config, Module) + do_check_config(Type, Config, Module) end. -do_check_conifg(Type, Config, Module) -> +do_check_config(Type, Config, Module) -> F = case erlang:function_exported(Module, check_config, 1) of true -> fun Module:check_config/1; false -> fun(C) -> - #{config := R} = - hocon_schema:check_plain(Module, #{<<"config">> => C}, + Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + R = hocon_schema:check_plain(Module, #{Key => C}, #{atom_key => true}), - R + maps:get(AtomKey, R) end end, try @@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M); authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)}; authn_type(#{<<"mechanism">> := M}) -> atom(M). -atom(Bin) -> - binary_to_existing_atom(Bin, utf8). +atom(A) when is_atom(A) -> A; +atom(Bin) -> binary_to_existing_atom(Bin, utf8). %% The relative dir for ssl files. certs_dir(ChainName, ConfigOrID) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 8806d4bc3..9979629bf 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -268,23 +268,39 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> +init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - %% check and save configs - {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0), + %% check configs agains the schema, with environment variables applied on top + {_AppEnvs, CheckedConf} = + check_config(SchemaMod, RawConf, #{apply_override_envs => true}), + %% fill default values for raw config + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(get_root_names(), RawConf0)). + maps:with(RootNames, RawConfWithEnvs)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. +merge_envs(SchemaMod, RawConf) -> + Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config + nullable => true, %% TODO: evil, remove, nullable should be declared in schema + format => map, + apply_override_envs => true + }, + hocon_schema:merge_env_overrides(SchemaMod, RawConf, all, Opts). + -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} when AppEnvs :: app_envs(), CheckedConf :: config(). check_config(SchemaMod, RawConf) -> - Opts = #{return_plain => true, - nullable => true, - format => map - }, + check_config(SchemaMod, RawConf, #{}). + +check_config(SchemaMod, RawConf, Opts0) -> + Opts1 = #{return_plain => true, + nullable => true, %% TODO: evil, remove, nullable should be declared in schema + format => map + }, + Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawConf, Opts), {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. @@ -312,13 +328,15 @@ read_override_conf(#{} = Opts) -> File = override_conf_file(Opts), load_hocon_file(File, map). -override_conf_file(Opts) -> +override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, local) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, - application:get_env(emqx, Key, undefined). + application:get_env(emqx, Key, undefined); +override_conf_file(Which) when is_atom(Which) -> + application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. save_schema_mod_and_names(SchemaMod) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e470ff478..1c471da90 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -22,6 +22,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include("emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type duration() :: integer(). @@ -105,11 +106,29 @@ and can not be deleted.""" The configs here work as default values which can be overriden in zone configs""" })} - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication( -"""Default authentication configs for all MQTT listeners.
+"""Default authentication configs for all MQTT listeners. +
For per-listener overrides see authentication -in listener configs""")} +in listener configs +
+
+EMQ X can be configured with: +
+ +
+When a chain is configured, the login credentials are checked against the backends +per the configured order, until an 'allow' or 'deny' decision can be made. +
+If there is no decision after a full chain exhaustion, the login is rejected. +""")} + %% NOTE: authorization schema here is only to keep emqx app prue + %% the full schema for EMQ X node is injected in emqx_conf_schema. , {"authorization", sc(ref("authorization"), #{})} @@ -972,7 +991,7 @@ mqtt_listener() -> sc(duration(), #{}) } - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication("Per-listener authentication override") } ]. @@ -1231,16 +1250,18 @@ ciphers_schema(Default) -> false -> fun validate_ciphers/1 end , desc => -"""TLS cipher suite names separated by comma, or as an array of strings +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. \"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or -[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"][\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"].
Ciphers (and their ordering) define the way in which the -client and server encrypts information over the wire. +client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the application's data security, confidentiality and performance. -The names should be in OpenSSL sting format (not RFC format). -Default values and examples proveded by EMQ X config + +The names should be in OpenSSL string format (not RFC format). +All default values and examples proveded by EMQ X config documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with @@ -1436,12 +1457,23 @@ str(S) when is_list(S) -> S. authentication(Desc) -> - #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])) - , desc => iolist_to_binary([Desc, "
", """ + %% authentication schemais lazy to make it more 'plugable' + %% the type checks are done in emqx_auth application when it boots. + %% and in emqx_authentication_config module for rutime changes. + Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])), + %% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY + %% is for now only affecting document generation. + %% maybe in the future, we can find a more straightforward way to support + %% * document generation (at compile time) + %% * type checks before boot (in bin/emqx config generation) + %% * type checks at runtime (when changing configs via management API) + #{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of + undefined -> Default; + Module -> hoconsc:lazy(Module:root_type()) + end + , desc => iolist_to_binary([Desc, """ Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
-EMQ X comes with a set of pre-built autenticators, for more details, see -autenticator_config """]) }. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 10a4e4091..2c28a6076 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -25,18 +25,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("typerefl/include/types.hrl"). - --export([ roots/0, fields/1 ]). - --export([ create/2 - , update/2 - , authenticate/2 - , destroy/1 - , check_config/1 - ]). +-include("emqx_authentication.hrl"). -define(AUTHN, emqx_authentication). -define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -250,7 +243,7 @@ t_update_config({init, Config}) -> {"auth2", AuthNType2} | Config]; t_update_config(Config) when is_list(Config) -> - emqx_config_handler:add_handler([authentication], emqx_authentication), + emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication), ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), @@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})), ?assertMatch( {ok, #{id := ID1, state := #{mark := 1}}}, @@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch( {ok, #{id := ID2, state := #{mark := 1}}}, @@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], + update_config([?CONF_ROOT], {update_authenticator, Global, ID1, @@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {move_authenticator, Global, ID2, top})), + update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})), ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), - ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})), ?assertEqual( {error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch( {ok, _}, - update_config([authentication], {delete_authenticator, Global, ID2})), + update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})), ?assertEqual( {error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)), ListenerID = 'tcp:default', - ConfKeyPath = [listeners, tcp, default, authentication], + ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT], ?assertMatch( {ok, _}, diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index d1d3d16f8..7503f3b89 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,6 +1 @@ -# authentication: { -# mechanism: password-based -# backend: built-in-database -# user_id_type: clientid -# } - +authentication: [] diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 1c5c00e85..b5dfa8f01 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -17,6 +17,8 @@ -ifndef(EMQX_AUTHN_HRL). -define(EMQX_AUTHN_HRL, true). +-include_lib("emqx/include/emqx_authentication.hrl"). + -define(APP, emqx_authn). -define(AUTHN, emqx_authentication). @@ -27,4 +29,9 @@ -define(AUTH_SHARD, emqx_authn_shard). +%% has to be the same as the root field name defined in emqx_schema +-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME). +-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). +-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -endif. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index fbd31c5d2..5b3d76822 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -22,6 +22,8 @@ , check_configs/1 ]). +-include("emqx_authn.hrl"). + providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} @@ -44,8 +46,8 @@ check_config(Config) -> check_config(Config, Opts) -> case do_check_config(Config, Opts) of - #{config := Checked} -> Checked; - #{<<"config">> := WithDefaults} -> WithDefaults + #{?CONF_NS_ATOM := Checked} -> Checked; + #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> @@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> case lists:keyfind(Key, 1, providers()) of false -> throw({unknown_handler, Key}); - {_, Provider} -> - hocon_schema:check_plain(Provider, #{<<"config">> => Config}, + {_, ProviderModule} -> + hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config}, Opts#{atom_key => true}) end. atom(Bin) -> - binary_to_existing_atom(Bin, utf8). + try + binary_to_existing_atom(Bin, utf8) + catch + _ : _ -> + throw({unknown_auth_provider, Bin}) + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 827b4e09c..595eed1c1 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,6 +22,7 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -import(hoconsc, [mk/2, ref/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -32,8 +33,10 @@ % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]). --define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]). +-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">>]). +-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">>]). -export([ api_spec/0 , paths/0 diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index df7fbecd3..035e61910 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -25,6 +25,8 @@ , stop/1 ]). +-include_lib("emqx/include/emqx_authentication.hrl"). + -dialyzer({nowarn_function, [start/2]}). %%------------------------------------------------------------------------------ @@ -65,7 +67,7 @@ chain_configs() -> [global_chain_config() | listener_chain_configs()]. global_chain_config() -> - {?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}. + {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}. listener_chain_configs() -> lists:map( @@ -77,7 +79,7 @@ listener_chain_configs() -> auth_config_path(ListenerID) -> [<<"listeners">>] ++ binary:split(atom_to_binary(ListenerID), <<":">>) - ++ [<<"authentication">>]. + ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY]. provider_types() -> lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 22f62f519..c2e963ec4 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,10 +22,12 @@ , roots/0 , fields/1 , authenticator_type/0 + , root_type/0 + , mechanism/1 + , backend/1 ]). -%% only for doc generation -roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}]. +roots() -> []. fields(_) -> []. @@ -35,6 +37,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(desc) -> "Set to false to disable this auth provider"; enable(_) -> undefined. authenticator_type() -> @@ -42,3 +45,18 @@ authenticator_type() -> config_refs(Modules) -> lists:append([Module:refs() || Module <- Modules]). + +%% authn is a core functionality however implemented outside fo emqx app +%% in emqx_schema, 'authentication' is a map() type which is to allow +%% EMQ X more plugable. +root_type() -> + T = authenticator_type(), + hoconsc:union([T, hoconsc:array(T)]). + +mechanism(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). + +backend(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index f02655462..3604455dc 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -83,11 +83,11 @@ mnesia(boot) -> namespace() -> "authn-scram-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, [scram]}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('scram')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). @@ -105,7 +105,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{algorithm := Algorithm, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 829ce2282..533301ea7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -43,8 +43,9 @@ namespace() -> "authn-http". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(get) -> @@ -60,8 +61,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, hoconsc:enum(['password-based'])} - , {backend, hoconsc:enum(['http'])} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(http)} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} @@ -233,9 +234,9 @@ transform_header_name(Headers) -> end, #{}, Headers). check_ssl_opts(Conf) -> - case parse_url(hocon_schema:get_value("config.url", Conf)) of + case parse_url(get_conf_val("url", Conf)) of #{scheme := https} -> - case hocon_schema:get_value("config.ssl.enable", Conf) of + case get_conf_val("ssl.enable", Conf) of true -> ok; false -> false end; @@ -244,8 +245,8 @@ check_ssl_opts(Conf) -> end. check_headers(Conf) -> - Method = to_bin(hocon_schema:get_value("config.method", Conf)), - Headers = hocon_schema:get_value("config.headers", Conf), + Method = to_bin(get_conf_val("method", Conf)), + Headers = get_conf_val("headers", Conf), Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). parse_url(URL) -> @@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L). + +get_conf_val(Name, Conf) -> + hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 916911ffa..9295e5c7e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -16,6 +16,7 @@ -module(emqx_authn_jwt). +-include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -40,9 +41,9 @@ namespace() -> "authn-jwt". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{} - )} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields('hmac-based') -> @@ -82,7 +83,7 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {mechanism, {enum, [jwt]}} + [ {mechanism, emqx_authn_schema:mechanism('jwt')} , {verify_claims, fun verify_claims/1} ] ++ emqx_authn_schema:common_fields(). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index fd02671fb..f609d8cac 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -85,11 +85,11 @@ mnesia(boot) -> namespace() -> "authn-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} ] ++ emqx_authn_schema:common_fields(); @@ -104,7 +104,7 @@ fields(other_algorithms) -> ]. user_id_type(type) -> user_id_type(); -user_id_type(default) -> username; +user_id_type(default) -> <<"username">>; user_id_type(_) -> undefined. password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), @@ -121,7 +121,7 @@ salt_rounds(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{user_id_type := Type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 2deef8506..3b47bcd7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -42,8 +42,8 @@ namespace() -> "authn-mongodb". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,8 +56,8 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mongodb]}} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mongodb)} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 47ca0ae3c..fd0d09f57 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -41,11 +41,11 @@ namespace() -> "authn-mysql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mysql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mysql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -74,7 +74,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 660acf566..fdd30b618 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -47,11 +47,11 @@ namespace() -> "authn-postgresql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [postgresql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(postgresql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -75,7 +75,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index d238bc537..e17d0ad8f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -42,8 +42,8 @@ namespace() -> "authn-redis". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,11 +56,11 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [{mechanism, {enum, ['password-based']}}, - {backend, {enum, [redis]}}, - {cmd, fun cmd/1}, - {password_hash_algorithm, fun password_hash_algorithm/1}, - {salt_position, fun salt_position/1} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(redis)} + , {cmd, fun cmd/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} ] ++ emqx_authn_schema:common_fields(). cmd(type) -> string(); diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index ba9f4996a..885811fec 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -45,11 +45,11 @@ groups() -> init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( - [authentication], + [?CONF_NS_ATOM], ?GLOBAL), emqx_authn_test_lib:delete_authenticators( - [listeners, tcp, default, authentication], + [listeners, tcp, default, ?CONF_NS_ATOM], ?TCP_DEFAULT), {atomic, ok} = mria:clear_table(emqx_authn_mnesia), @@ -89,8 +89,8 @@ set_special_configs(_App) -> %%------------------------------------------------------------------------------ t_invalid_listener(_) -> - {ok, 404, _} = request(get, uri(["listeners", "invalid", "authentication"])), - {ok, 404, _} = request(get, uri(["listeners", "in:valid", "authentication"])). + {ok, 404, _} = request(get, uri(["listeners", "invalid", ?CONF_NS])), + {ok, 404, _} = request(get, uri(["listeners", "in:valid", ?CONF_NS])). t_authenticators(_) -> test_authenticators([]). @@ -133,86 +133,86 @@ test_authenticators(PathPrefix) -> ValidConfig = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig), {ok, 409, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig), InvalidConfig0 = ValidConfig#{method => <<"delete">>}, {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), InvalidConfig0), InvalidConfig1 = ValidConfig#{method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>}}, {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), InvalidConfig1), ?assertAuthenticatorsMatch( [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], - PathPrefix ++ ["authentication"]). + PathPrefix ++ [?CONF_NS]). test_authenticator(PathPrefix) -> ValidConfig0 = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig0), {ok, 200, _} = request( get, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"])), {ok, 404, _} = request( get, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])), {ok, 404, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database"]), emqx_authn_test_lib:built_in_database_example()), InvalidConfig0 = ValidConfig0#{method => <<"delete">>}, {ok, 400, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), InvalidConfig0), InvalidConfig1 = ValidConfig0#{method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>}}, {ok, 400, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), InvalidConfig1), ValidConfig1 = ValidConfig0#{pool_size => 9}, {ok, 200, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), ValidConfig1), {ok, 404, _} = request( delete, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])), {ok, 204, _} = request( delete, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"])), - ?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]). + ?assertAuthenticatorsMatch([], PathPrefix ++ [?CONF_NS]). test_authenticator_users(PathPrefix) -> - UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), InvalidUsers = [ @@ -263,11 +263,11 @@ test_authenticator_users(PathPrefix) -> lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])). test_authenticator_user(PathPrefix) -> - UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), User = #{user_id => <<"u1">>, password => <<"p1">>}, @@ -311,7 +311,7 @@ test_authenticator_move(PathPrefix) -> fun(Conf) -> {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), Conf) end, AuthenticatorConfs), @@ -322,40 +322,40 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"jwt">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), % Invalid moves {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"up">>}), {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:invalid">>}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:redis">>}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:redis">>}), % Valid moves {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"top">>}), ?assertAuthenticatorsMatch( @@ -364,11 +364,11 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"bottom">>}), ?assertAuthenticatorsMatch( @@ -377,11 +377,11 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}, #{<<"mechanism">> := <<"jwt">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:built-in-database">>}), ?assertAuthenticatorsMatch( @@ -390,17 +390,17 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"jwt">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]). + PathPrefix ++ [?CONF_NS]). test_authenticator_import_users(PathPrefix) -> ImportUri = uri( PathPrefix ++ - ["authentication", "password-based:built-in-database", "import_users"]), + [?CONF_NS, "password-based:built-in-database", "import_users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), {ok, 400, _} = request(post, ImportUri, #{}), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 67865d645..2c0716e8b 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --define(PATH, [authentication]). +-define(PATH, [?CONF_NS_ATOM]). -define(HTTP_PORT, 33333). -define(HTTP_PATH, "/auth"). diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 5c6eb694c..53c8016cd 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -49,6 +49,8 @@ end_per_testcase(_Case, Config) -> %% Tests %%------------------------------------------------------------------------------ +-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}). + t_check_schema(_Config) -> ConfigOk = #{ <<"mechanism">> => <<"password-based">>, @@ -60,7 +62,7 @@ t_check_schema(_Config) -> } }, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}), + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)), ConfigNotOk = #{ <<"mechanism">> => <<"password-based">>, @@ -74,7 +76,7 @@ t_check_schema(_Config) -> ?assertException( throw, {emqx_authn_mnesia, _}, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})). + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))). t_create(_) -> Config0 = config(), diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 49056ed68..2bca1793d 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -31,7 +31,7 @@ groups() -> init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], + [emqx_connector, emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 417fa49f7..6e274903e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -33,7 +33,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 26a1d5bd1..3a3151ef0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -108,7 +108,8 @@ connector_name() -> #{ nullable => false , desc =>""" The connector name to be used for this bridge. -Connectors are configured by 'connectors.. +Connectors are configured as 'connectors.type.name', +for example 'connectors.http.mybridge'. """ })}. diff --git a/apps/emqx_conf/etc/emqx_conf.md b/apps/emqx_conf/etc/emqx_conf.md new file mode 100644 index 000000000..f68411bff --- /dev/null +++ b/apps/emqx_conf/etc/emqx_conf.md @@ -0,0 +1,192 @@ +EMQ X configuration file is in [HOCON](https://github.com/emqx/hocon) format. +HOCON, or Human-Optimized Config Object Notation is a format for human-readable data, +and a superset of JSON. + +## Syntax + +In config file the values can be notated as JSON like ojbects, such as +``` +node { + name = "emqx@127.0.0.1" + cookie = "mysecret" +} +``` + +Another equivalent representation is flat, suh as + +``` +node.name="127.0.0.1" +node.cookie="mysecret" +``` + +This flat format is almost backward compatible with EMQ X's config file format +in 4.x series (the so called 'cuttlefish' format). + +It is 'almost' compabile because the often HOCON requires strings to be quoted, +while cuttlefish treats all characters to the right of the `=` mark as the value. + +e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"` + +Strings without special characters in them can be unquoted in HOCON too, +e.g. `foo`, `foo_bar`, `foo_bar_1`: + +For more HOCON syntax, pelase refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md) + +## Schema + +To make the HOCON objects type-safe, EMQ X introduded a schema for it. +The schema defines data types, and data fields' names and metadata for config value validation +and more. In fact, this config document itself is generated from schema metadata. + +### Complex Data Types + +There are 4 complex data types in EMQ X's HOCON config: + +1. Struct: Named using an unquoted string, followed by a pre-defined list of fields, + fields can not start with a number, and are only allowed to use + lowercase letters and underscores as word separater. +1. Map: Map is like Struct, however the fields are not pre-defined. + 1-based index number can also be used as map keys for an alternative + representation of an Array. +1. Union: `MemberType1 | MemberType2 | ...` +1. Array: `[ElementType]` + +### Primitive Data Types + +Complex types define data 'boxes' wich may contain other complex data +or primitive values. +There are quite some different primitive types, to name a fiew: + +* `atom()` +* `boolean()` +* `string()` +* `integer()` +* `float()` +* `number()` +* `binary()` # another format of string() +* `emqx_schema:duration()` # time duration, another format of integer() +* ... + +The primitive types are mostly self-describing, some are built-in, such +as `atom()`, some are defiend in EMQ X modules, such as `emqx_schema:duration()`. + +### Config Paths + +If we consider the whole EMQ X config as a tree, +to reference a primitive value, we can use a dot-separated names form string for +the path from the tree-root (always a Struct) down to the primitive values at tree-leaves. + +Each segment of the dotted string is a Struct filed name or Map key. +For Array elements, 1-based index is used. + +below are some examples + +``` +node.name="emqx.127.0.0.1" +zone.zone1.max_packet_size="10M" +authentication.1.enable=true +``` + +### Environment varialbes + +Environment variables can be used to define or override config values. + +Due to the fact that dots (`.`) are not allowed in environment variables, dots are +replaced with double-underscores (`__`). + +And a the `EMQX_` prefix is used as the namespace. + +For example `node.name` can be represented as `EMQX_NODE__NAME` + +Environment varialbe values are parsed as hocon values, this allows users +to even set complex values from environment variables. + +For example, this environment variable sets an array value. + +``` +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +``` + +Unknown environment variables are logged as a `warning` level log, for example: + +``` +[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"] +``` + +because the field name is `enable`, not `enabled`. + +NOTE: Unknown root keys are however silently discarded. + +### Config overlay + +HOCON values are overlayed, earlier defined values are at layers closer to the bottom. +The overall order of the overlay rules from bottom up are: + +1. `emqx.conf` the base config file +1. `EMQX_` prfixed environment variables +1. Cluster override file, the path of which is configured as `cluster_override_conf_file` in the lower layers +1. Local override file, the path of which is configured as `local_override_conf_file` in the lower layers + +Below are the rules of config value overlay. + +#### Struct Fileds + +Later config values overwrites earlier values. +For example, in below config, the last line `debug` overwrites `errro` for +console log handler's `level` config, but leaving `enable` unchanged. +``` +log { + console_handler{ + enable=true, + level=error + } +} + +## ... more configs ... + +log.console_handler.level=debug +``` + +#### Map Values + +Maps are like structs, only the files are user-defined rather than +the config schema. For instance, `zone1` in the exampele below. + +``` +zone { + zone1 { + mqtt.max_packet_size = 1M + } +} + +## The maximum packet size can be defined as above, +## then overriden as below + +zone.zone1.mqtt.max_packet_size = 10M +``` + +#### Array Elements + +Arrays in EMQ X config have two different representations + +* list, such as: `[1, 2, 3]` +* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}` + +Dot-separated paths with number in it are parsed to indexed-maps +e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}` + +Indexed-map arrays can be used to override list arrays: + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +# we can disable this authentication provider with: +authentication.1.enable=false +``` +However, list arrays do not get recursively merged into indexed-map arrays. +e.g. + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +## below value will replace the whole array, but not to override just one field. +authentication=[{enable=true}] +``` diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index c82623e72..dec07f35c 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -24,6 +24,7 @@ -export([update/3, update/4]). -export([remove/2, remove/3]). -export([reset/2, reset/3]). +-export([gen_doc/1]). %% for rpc -export([get_node_and_config/1]). @@ -123,6 +124,16 @@ reset(Node, KeyPath, Opts) when Node =:= node() -> reset(Node, KeyPath, Opts) -> rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]). +-spec gen_doc(file:name_all()) -> ok. +gen_doc(File) -> + Version = emqx_release:version(), + Title = "# EMQ X " ++ Version ++ " Configuration", + BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]), + {ok, Body} = file:read_file(BodyFile), + Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title, + body => Body}), + file:write_file(File, Doc). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 617703cd2..8103796b7 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -24,6 +24,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -62,8 +63,12 @@ namespace() -> undefined. roots() -> - %% authorization configs are merged in THIS schema's "authorization" fields - lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++ + PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, + case persistent_term:get(PtKey, undefined) of + undefined -> persistent_term:put(PtKey, emqx_authn_schema); + _ -> ok + end, + emqx_schema_high_prio_roots() ++ [ {"node", sc(hoconsc:ref("node"), #{ desc => "Node name, cookie, config & data directories " @@ -87,20 +92,6 @@ roots() -> "should work, but in case you need to do performance " "fine-turning or experiment a bit, this is where to look." })} - , {"authorization", - sc(hoconsc:ref("authorization"), - #{ desc => """ -Authorization a.k.a ACL.
-In EMQ X, MQTT client access control is extremly flexible.
-An out of the box set of authorization data sources are supported. -For example,
-'file' source is to support concise and yet generic ACL rules in a file;
-'built-in-database' source can be used to store per-client customisable rule sets, -natively in the EMQ X node;
-'http' source to make EMQ X call an external HTTP API to make the decision;
-'postgresql' etc. to look up clients or rules from external databases;
-""" - })} , {"db", sc(ref("db"), #{ desc => "Settings of the embedded database." @@ -251,14 +242,12 @@ fields("node") -> [ {"name", sc(string(), #{ default => "emqx@127.0.0.1" - , override_env => "EMQX_NODE_NAME" })} , {"cookie", sc(string(), #{ mapping => "vm_args.-setcookie", default => "emqxsecretcookie", - sensitive => true, - override_env => "EMQX_NODE_COOKIE" + sensitive => true })} , {"data_dir", sc(string(), @@ -845,3 +834,22 @@ ensure_list(V) -> roots(Module) -> lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)). + +%% Like authentication schema, authorization schema is incomplete in emqx_schema +%% module, this function replaces the root filed "authorization" with a new schema +emqx_schema_high_prio_roots() -> + Roots = emqx_schema:roots(high), + Authz = {"authorization", + sc(hoconsc:ref("authorization"), + #{ desc => """ +Authorization a.k.a ACL.
+In EMQ X, MQTT client access control is extremly flexible.
+An out of the box set of authorization data sources are supported. +For example,
+'file' source is to support concise and yet generic ACL rules in a file;
+'built-in-database' source can be used to store per-client customisable rule sets, +natively in the EMQ X node;
+'http' source to make EMQ X call an external HTTP API to make the decision;
+'postgresql' etc. to look up clients or rules from external databases;
+""" })}, + lists:keyreplace("authorization", 1, Roots, Authz). diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 6eb397519..7e934b997 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -38,7 +38,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). namespace() -> "connector". @@ -74,7 +74,7 @@ schema("/connectors_test") -> post => #{ tags => [<<"connectors">>], description => <<"Test creating a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Test creating connector">>, requestBody => connector_test_info(), responses => #{ @@ -98,7 +98,7 @@ schema("/connectors") -> post => #{ tags => [<<"connectors">>], description => <<"Create a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Create connector">>, requestBody => connector_info(), responses => #{ diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index 264a6dbd6..3caf2b595 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -11,6 +11,7 @@ roots() -> ["connectors"]. +fields(connectors) -> fields("connectors"); fields("connectors") -> [ {mqtt, sc(hoconsc:map(name, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index a699da8e6..a722872a3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -182,12 +182,12 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, case hocon_schema:field_schema(Type, in) of path -> - Option = #{atom_key => true, override_env => false}, + Option = #{atom_key => true}, NewBindings = hocon_schema:check_plain(Schema, Bindings, Option), NewBindingsAcc = maps:merge(BindingsAcc, NewBindings), check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc); query -> - Option = #{override_env => false}, + Option = #{}, NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option), NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc) @@ -201,7 +201,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) -> _ -> Type0 end, NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]}, - Option = #{override_env => false, nullable => true}, + Option = #{nullable => true}, #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option), NewBody; %% TODO not support nest object check yet, please use ref! @@ -214,7 +214,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) -> check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) -> lists:foldl(fun({Name, Type}, Acc) -> Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, - maps:merge(Acc, CheckFun(Schema, Body, #{override_env => false})) + maps:merge(Acc, CheckFun(Schema, Body, #{})) end, #{}, Spec). %% tags, description, summary, security, deprecated diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index d7b0a00c3..e1dad1dee 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -18,9 +18,6 @@ -behaviour(emqx_gateway_channel). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). - %% API -export([ info/1 , info/2 @@ -44,6 +41,12 @@ -export_type([channel/0]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + -record(channel, { %% Context ctx :: emqx_gateway_ctx:context(), @@ -283,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) -> %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - case emqx_conf:get([gateway, coap, authentication], undefined) of + case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of undefined -> call_session(handle_request, Msg, Channel); _ -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index d79a880a1..925b7cc0e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -17,6 +17,7 @@ -module(emqx_gateway_api). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -behaviour(minirest_api). @@ -243,7 +244,7 @@ schema_gateway_overview_list() -> %% %% NOTE: It is a temporary measure to generate swagger-schema -define(COAP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, @@ -331,7 +332,7 @@ schema_gateway_overview_list() -> ). -define(STOMP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 3612b428d..e97b0062d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -17,8 +17,6 @@ %% @doc The gateway configuration management module -module(emqx_gateway_conf). --include_lib("emqx/include/logger.hrl"). - %% Load/Unload -export([ load/0 , unload/0 @@ -56,6 +54,10 @@ , post_config_update/5 ]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -type atom_or_bin() :: atom() | binary(). -type ok_or_err() :: ok_or_err(). -type listener_ref() :: {ListenerType :: atom_or_bin(), @@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) -> -spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). update_gateway(GwName, Conf0) -> - Conf = maps:without([listeners, authentication, - <<"listeners">>, <<"authentication">>], Conf0), + Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], + Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0], + Conf = maps:without(Exclude0 ++ Exclude1, Conf0), update({?FUNCTION_NAME, bin(GwName), Conf}). %% FIXME: delete cert files ?? @@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> undefined -> {error, not_found}; _ -> - NConf = maps:without([<<"listeners">>, - <<"authentication">>], Conf), + NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> @@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})}; + #{GwName => #{?AUTHN_BIN => Conf}})}; _ -> {error, already_exist} end; @@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(<<"authentication">>, Conf, Listener), + NListener = maps:put(?AUTHN_BIN, Conf, Listener), NGateway = #{GwName => #{<<"listeners">> => #{LType => #{LName => NListener}}}}, @@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {error, not_found}; _ -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})} + #{GwName => #{?AUTHN_BIN => Conf}})} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case emqx_map_lib:deep_get( @@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> {error, not_found}; Auth -> NListener = maps:put( - <<"authentication">>, + ?AUTHN_BIN, emqx_map_lib:deep_merge(Auth, Conf), Listener ), @@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, emqx_map_lib:deep_remove( - [GwName, <<"authentication">>], RawConf)}; + [GwName, ?AUTHN_BIN], RawConf)}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> - Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>], + Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], {ok, emqx_map_lib:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 0d2f765c5..98344f968 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -19,6 +19,9 @@ -include("include/emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %% Mgmt APIs - gateway -export([ gateways/1 @@ -166,7 +169,7 @@ remove_listener(ListenerId) -> -spec authn(gateway_name()) -> map(). authn(GwName) -> %% XXX: Need append chain-nanme, authenticator-id? - Path = [gateway, GwName, authentication], + Path = [gateway, GwName, ?AUTHN], ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, @@ -176,7 +179,7 @@ authn(GwName) -> -spec authn(gateway_name(), binary()) -> map(). authn(GwName, ListenerId) -> {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), - Path = [gateway, GwName, listeners, Type, Name, authentication], + Path = [gateway, GwName, listeners, Type, Name, ?AUTHN], ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9f35225b7..5f87131d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -24,6 +24,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type ip_port() :: tuple(). @@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW" , desc => "The Pre-defined topic ids and topic names.
A 'pre-defined' topic id is a topic id whose mapping to a topic name -is known in advance by both the client’s application and the gateway" +is known in advance by both the client's application and the gateway" })} , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); @@ -407,30 +408,14 @@ fields(dtls_opts) -> , ciphers => dtls_all_available }, false). -authentication() -> - sc(hoconsc:union( - [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]), - #{ nullable => {true, recursively} - , desc => +authentication_schema() -> + sc(emqx_authn_schema:authenticator_type(), + #{ nullable => {true, recursively} + , desc => """Default authentication configs for all of the gateway listeners.
For per-listener overrides see authentication in listener configs""" - }). + }). gateway_common_options() -> [ {enable, @@ -464,7 +449,7 @@ it has two purposes: sc(ref(clientinfo_override), #{ desc => "" })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} ]. common_listener_opts() -> @@ -483,7 +468,7 @@ common_listener_opts() -> sc(integer(), #{ default => 1000 })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} , {mountpoint, sc(binary(), #{ default => undefined diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index e49a78e73..8e25a4e6e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -28,6 +28,8 @@ -include_lib("eunit/include/eunit.hrl"). +%% this parses to #{}, will not cause config cleanup +%% so we will need call emqx_config:erase -define(CONF_DEFAULT, <<" gateway {} ">>). @@ -39,6 +41,7 @@ gateway {} all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_config:erase(gateway), emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Conf. diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 713468460..27093ef1d 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -170,9 +170,9 @@ t_update_re_failed(_Config) -> {error, {emqx_modules_schema, [{validation_error, - #{array_index => 1,path => "rewrite.re", - reason => {<<"*^test/*">>,{"nothing to repeat",0}}, - value => <<"*^test/*">>}}]}}}, + #{path => "rewrite.1.re", + reason => {<<"*^test/*">>,{"nothing to repeat",0}}, + value => <<"*^test/*">>}}]}}}, ?assertError(Error, emqx_rewrite:update(Rules)), ok. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 8daae99b5..995044fc7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -56,7 +56,7 @@ A list of outputs of the rule.
An output can be a string that refers to the channel Id of a emqx bridge, or a object that refers to a function.
There a some built-in functions like \"republish\" and \"console\", and we also support user -provided functions like \":\".
+provided functions like \"ModuleName:FunctionName\".
The outputs in the list is executed one by one in order. This means that if one of the output is executing slowly, all of the outputs comes after it will not be executed until it returns.
diff --git a/bin/emqx b/bin/emqx index e1b72d1f9..7862a98b8 100755 --- a/bin/emqx +++ b/bin/emqx @@ -342,6 +342,9 @@ generate_config() { NOW_TIME="$(call_hocon now_time)" ## ths command populates two files: app.