Merge pull request #6362 from zmstone/emqx-config-put-raw-with-env-vars

fix: config put raw with env vars

Makefile

@@ -57,7 +57,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct:
+$1-ct: conf-segs
 	$(REBAR) ct --name $(CT_NODE_NAME) -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))

apps/emqx/include/emqx_authentication.hrl

@@ -0,0 +1,31 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_AUTHENTICATION_HRL).
+-define(EMQX_AUTHENTICATION_HRL, true).
+
+%% config root name all auth providers have to agree on.
+-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication").
+-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication).
+-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>).
+
+%% key to a persistent term which stores a module name in order to inject
+%% schema module at run-time to keep emqx app's compile time purity.
+%% see emqx_schema.erl for more details
+%% and emqx_conf_schema for an examples
+-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module).
+
+-endif.

apps/emqx/rebar.config

@@ -17,7 +17,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}}

apps/emqx/src/emqx_access_control.erl

@@ -63,4 +63,5 @@ do_authorize(ClientInfo, PubSub, Topic) ->
 
 -compile({inline, [run_hooks/3]}).
 run_hooks(Name, Args, Acc) ->
-    ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc).
+    ok = emqx_metrics:inc(Name),
+    emqx_hooks:run_fold(Name, Args, Acc).

apps/emqx/src/emqx_authentication.erl

@@ -24,9 +24,12 @@
 
 -include("emqx.hrl").
 -include("logger.hrl").
+-include("emqx_authentication.hrl").
 
 -include_lib("stdlib/include/ms_transform.hrl").
 
+-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
+
 %% The authentication entrypoint.
 -export([ authenticate/2
         ]).
@@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) ->
 %%--------------------------------------------------------------------
 
 init(_Opts) ->
-    ok = emqx_config_handler:add_handler([authentication], ?MODULE),
+    ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE),
-    ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE),
+    ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE),
     {ok, #{hooked => false, providers => #{}}}.
 
 handle_call(get_providers, _From, #{providers := Providers} = State) ->
@@ -496,8 +499,8 @@ terminate(Reason, _State) ->
         Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating",
                                 reason => Other})
     end,
-    emqx_config_handler:remove_handler([authentication]),
+    emqx_config_handler:remove_handler([?CONF_ROOT]),
-    emqx_config_handler:remove_handler([listeners, '?', '?', authentication]),
+    emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->

apps/emqx/src/emqx_authentication_config.erl

@@ -34,6 +34,7 @@
 -export_type([config/0]).
 
 -include("logger.hrl").
+-include("emqx_authentication.hrl").
 
 -type parsed_config() :: #{mechanism := atom(),
                            backend => atom(),
@@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}
 
 check_configs(Configs) ->
     Providers = emqx_authentication:get_providers(),
-    lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs).
+    lists:map(fun(C) -> do_check_config(C, Providers) end, Configs).
 
-do_check_conifg(Config, Providers) ->
+do_check_config(Config, Providers) ->
     Type = authn_type(Config),
     case maps:get(Type, Providers, false) of
         false ->
@@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) ->
                              providers => Providers}),
             throw({unknown_authn_type, Type});
         Module ->
-            do_check_conifg(Type, Config, Module)
+            do_check_config(Type, Config, Module)
     end.
 
-do_check_conifg(Type, Config, Module) ->
+do_check_config(Type, Config, Module) ->
     F = case erlang:function_exported(Module, check_config, 1) of
             true ->
                 fun Module:check_config/1;
             false ->
                 fun(C) ->
-                        #{config := R} =
+                        Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
-                            hocon_schema:check_plain(Module, #{<<"config">> => C},
+                        AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
+                        R = hocon_schema:check_plain(Module, #{Key => C},
                                                      #{atom_key => true}),
-                        R
+                        maps:get(AtomKey, R)
                 end
         end,
     try
@@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M);
 authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)};
 authn_type(#{<<"mechanism">> := M}) -> atom(M).
 
-atom(Bin) ->
+atom(A) when is_atom(A) -> A;
-    binary_to_existing_atom(Bin, utf8).
+atom(Bin) -> binary_to_existing_atom(Bin, utf8).
 
 %% The relative dir for ssl files.
 certs_dir(ChainName, ConfigOrID) ->

apps/emqx/src/emqx_config.erl

@@ -268,23 +268,39 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
                           }),
             error(failed_to_load_hocon_conf)
     end;
-init_load(SchemaMod, RawConf0) when is_map(RawConf0) ->
+init_load(SchemaMod, RawConf) when is_map(RawConf) ->
     ok = save_schema_mod_and_names(SchemaMod),
-    %% check and save configs
+    %% check configs agains the schema, with environment variables applied on top
-    {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0),
+    {_AppEnvs, CheckedConf} =
+        check_config(SchemaMod, RawConf, #{apply_override_envs => true}),
+    %% fill default values for raw config
+    RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
+    RootNames = get_root_names(),
     ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf),
-            maps:with(get_root_names(), RawConf0)).
+                            maps:with(RootNames, RawConfWithEnvs)).
 
 include_dirs() ->
     [filename:join(emqx:data_dir(), "configs")].
 
+merge_envs(SchemaMod, RawConf) ->
+    Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config
+             nullable => true, %% TODO: evil, remove, nullable should be declared in schema
+             format => map,
+             apply_override_envs => true
+            },
+    hocon_schema:merge_env_overrides(SchemaMod, RawConf, all, Opts).
+
 -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf}
     when AppEnvs :: app_envs(), CheckedConf :: config().
 check_config(SchemaMod, RawConf) ->
-    Opts = #{return_plain => true,
+    check_config(SchemaMod, RawConf, #{}).
-             nullable => true,
+
-             format => map
+check_config(SchemaMod, RawConf, Opts0) ->
-            },
+    Opts1 = #{return_plain => true,
+              nullable => true, %% TODO: evil, remove, nullable should be declared in schema
+              format => map
+             },
+    Opts = maps:merge(Opts0, Opts1),
     {AppEnvs, CheckedConf} =
         hocon_schema:map_translate(SchemaMod, RawConf, Opts),
     {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}.
@@ -312,13 +328,15 @@ read_override_conf(#{} = Opts) ->
     File = override_conf_file(Opts),
     load_hocon_file(File, map).
 
-override_conf_file(Opts) ->
+override_conf_file(Opts) when is_map(Opts) ->
     Key =
         case maps:get(override_to, Opts, local) of
             local -> local_override_conf_file;
             cluster -> cluster_override_conf_file
         end,
-    application:get_env(emqx, Key, undefined).
+    application:get_env(emqx, Key, undefined);
+override_conf_file(Which) when is_atom(Which) ->
+    application:get_env(emqx, Which, undefined).
 
 -spec save_schema_mod_and_names(module()) -> ok.
 save_schema_mod_and_names(SchemaMod) ->

apps/emqx/src/emqx_schema.erl

@@ -22,6 +22,7 @@
 -dialyzer(no_unused).
 -dialyzer(no_fail_call).
 
+-include("emqx_authentication.hrl").
 -include_lib("typerefl/include/types.hrl").
 
 -type duration() :: integer().
@@ -105,11 +106,29 @@ and can not be deleted."""
 The configs here work as default values which can be overriden
 in <code>zone</code> configs"""
           })}
-    , {"authentication",
+    , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,
        authentication(
-"""Default authentication configs for all MQTT listeners.<br>
+"""Default authentication configs for all MQTT listeners.
+<br>
 For per-listener overrides see <code>authentication</code>
-in listener configs""")}
+in listener configs
+<br>
+<br>
+EMQ X can be configured with:
+<br>
+<ul>
+<li><code>[]</code>: The default value, it allows *ALL* logins</li>
+<li>one: For example <code>{enable:true,backend:\"built-in-database\",mechanism=\"password-based\"}</code></li>
+<li>chain: An array of structs.</li>
+</ul>
+<br>
+When a chain is configured, the login credentials are checked against the backends
+per the configured order, until an 'allow' or 'deny' decision can be made.
+<br>
+If there is no decision after a full chain exhaustion, the login is rejected.
+""")}
+    %% NOTE: authorization schema here is only to keep emqx app prue
+    %% the full schema for EMQ X node is injected in emqx_conf_schema.
     , {"authorization",
        sc(ref("authorization"),
           #{})}
@@ -972,7 +991,7 @@ mqtt_listener() ->
        sc(duration(),
           #{})
       }
-    , {"authentication",
+    , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,
        authentication("Per-listener authentication override")
       }
     ].
@@ -1231,16 +1250,18 @@ ciphers_schema(Default) ->
                            false -> fun validate_ciphers/1
                        end
         , desc =>
-"""TLS cipher suite names separated by comma, or as an array of strings
+"""This config holds TLS cipher suite names separated by comma,
+or as an array of strings. e.g.
 <code>\"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\"</code> or
-<code>[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"]</code].
+<code>[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"]</code>.
 <br>
 Ciphers (and their ordering) define the way in which the
-client and server encrypts information over the wire.
+client and server encrypts information over the network connection.
 Selecting a good cipher suite is critical for the
 application's data security, confidentiality and performance.
-The names should be in OpenSSL sting format (not RFC format).
+
-Default values and examples proveded by EMQ X config
+The names should be in OpenSSL string format (not RFC format).
+All default values and examples proveded by EMQ X config
 documentation are all in OpenSSL format.<br>
 
 NOTE: Certain cipher suites are only compatible with
@@ -1436,12 +1457,23 @@ str(S) when is_list(S) ->
     S.
 
 authentication(Desc) ->
-    #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())]))
+    %% authentication schemais lazy to make it more 'plugable'
-     , desc => iolist_to_binary([Desc, "<br>", """
+    %% the type checks are done in emqx_auth application when it boots.
+    %% and in emqx_authentication_config module for rutime changes.
+    Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])),
+    %% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY
+    %% is for now only affecting document generation.
+    %% maybe in the future, we can find a more straightforward way to support
+    %% * document generation (at compile time)
+    %% * type checks before boot (in bin/emqx config generation)
+    %% * type checks at runtime (when changing configs via management API)
+    #{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of
+                   undefined -> Default;
+                   Module -> hoconsc:lazy(Module:root_type())
+               end
+     , desc => iolist_to_binary([Desc, """
 Authentication can be one single authenticator instance or a chain of authenticators as an array.
 When authenticating a login (username, client ID, etc.) the authenticators are checked
 in the configured order.<br>
-EMQ X comes with a set of pre-built autenticators, for more details, see
-<a href=\"#root-authenticator_config\">autenticator_config<a>
 """])
      }.

apps/emqx/test/emqx_authentication_SUITE.erl

@@ -25,18 +25,11 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("typerefl/include/types.hrl").
-
+-include("emqx_authentication.hrl").
--export([ roots/0, fields/1 ]).
-
--export([ create/2
-        , update/2
-        , authenticate/2
-        , destroy/1
-        , check_config/1
-        ]).
 
 -define(AUTHN, emqx_authentication).
 -define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()).
+-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
 
 %%------------------------------------------------------------------------------
 %% Hocon Schema
@@ -250,7 +243,7 @@ t_update_config({init, Config}) ->
      {"auth2", AuthNType2} | Config];
 
 t_update_config(Config) when is_list(Config) ->
-    emqx_config_handler:add_handler([authentication], emqx_authentication),
+    emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication),
     ok = register_provider(?config("auth1"), ?MODULE),
     ok = register_provider(?config("auth2"), ?MODULE),
     Global = ?config(global),
@@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) ->
 
     ?assertMatch(
        {ok, _},
-       update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})),
+       update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})),
 
     ?assertMatch(
        {ok, #{id := ID1, state := #{mark := 1}}},
@@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) ->
 
     ?assertMatch(
        {ok, _},
-       update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})),
+       update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})),
 
     ?assertMatch(
        {ok, #{id := ID2, state := #{mark := 1}}},
@@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) ->
 
     ?assertMatch(
        {ok, _},
-       update_config([authentication],
+       update_config([?CONF_ROOT],
                      {update_authenticator,
                       Global,
                       ID1,
@@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) ->
 
     ?assertMatch(
        {ok, _},
-       update_config([authentication], {move_authenticator, Global, ID2, top})),
+       update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})),
 
     ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)),
 
-    ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})),
+    ?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})),
     ?assertEqual(
        {error, {not_found, {authenticator, ID1}}},
        ?AUTHN:lookup_authenticator(Global, ID1)),
 
     ?assertMatch(
        {ok, _},
-       update_config([authentication], {delete_authenticator, Global, ID2})),
+       update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})),
 
     ?assertEqual(
        {error, {not_found, {authenticator, ID2}}},
        ?AUTHN:lookup_authenticator(Global, ID2)),
 
     ListenerID = 'tcp:default',
-    ConfKeyPath = [listeners, tcp, default, authentication],
+    ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT],
 
     ?assertMatch(
        {ok, _},

apps/emqx_authn/etc/emqx_authn.conf

@@ -1,6 +1 @@
-# authentication: {
+authentication: []
-#     mechanism: password-based
-#     backend: built-in-database
-#     user_id_type: clientid
-# }
-

apps/emqx_authn/include/emqx_authn.hrl

@@ -17,6 +17,8 @@
 -ifndef(EMQX_AUTHN_HRL).
 -define(EMQX_AUTHN_HRL, true).
 
+-include_lib("emqx/include/emqx_authentication.hrl").
+
 -define(APP, emqx_authn).
 
 -define(AUTHN, emqx_authentication).
@@ -27,4 +29,9 @@
 
 -define(AUTH_SHARD, emqx_authn_shard).
 
+%% has to be the same as the root field name defined in emqx_schema
+-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME).
+-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
+-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY).
+
 -endif.

apps/emqx_authn/src/emqx_authn.erl

@@ -22,6 +22,8 @@
         , check_configs/1
         ]).
 
+-include("emqx_authn.hrl").
+
 providers() ->
     [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia}
     , {{'password-based', mysql}, emqx_authn_mysql}
@@ -44,8 +46,8 @@ check_config(Config) ->
 
 check_config(Config, Opts) ->
     case do_check_config(Config, Opts) of
-        #{config := Checked} -> Checked;
+        #{?CONF_NS_ATOM := Checked} -> Checked;
-        #{<<"config">> := WithDefaults} -> WithDefaults
+        #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults
     end.
 
 do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
@@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
     case lists:keyfind(Key, 1, providers()) of
         false ->
             throw({unknown_handler, Key});
-        {_, Provider} ->
+        {_, ProviderModule} ->
-            hocon_schema:check_plain(Provider, #{<<"config">> => Config},
+            hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config},
                                      Opts#{atom_key => true})
     end.
 
 atom(Bin) ->
-    binary_to_existing_atom(Bin, utf8).
+    try
+        binary_to_existing_atom(Bin, utf8)
+    catch
+        _ : _ ->
+            throw({unknown_auth_provider, Bin})
+    end.

apps/emqx_authn/src/emqx_authn_api.erl

@@ -22,6 +22,7 @@
 -include("emqx_authn.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
 
 -import(hoconsc, [mk/2, ref/1]).
 -import(emqx_dashboard_swagger, [error_codes/2]).
@@ -32,8 +33,10 @@
 
 % Swagger
 
--define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]).
+-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
--define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]).
+                          <<"authentication config(global)">>]).
+-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
+                          <<"authentication config(single listener)">>]).
 
 -export([ api_spec/0
         , paths/0

apps/emqx_authn/src/emqx_authn_app.erl

@@ -25,6 +25,8 @@
         , stop/1
         ]).
 
+-include_lib("emqx/include/emqx_authentication.hrl").
+
 -dialyzer({nowarn_function, [start/2]}).
 
 %%------------------------------------------------------------------------------
@@ -65,7 +67,7 @@ chain_configs() ->
     [global_chain_config() | listener_chain_configs()].
 
 global_chain_config() ->
-    {?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}.
+    {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}.
 
 listener_chain_configs() ->
     lists:map(
@@ -77,7 +79,7 @@ listener_chain_configs() ->
 auth_config_path(ListenerID) ->
     [<<"listeners">>]
     ++ binary:split(atom_to_binary(ListenerID), <<":">>)
-    ++ [<<"authentication">>].
+    ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY].
 
 provider_types() ->
     lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()).

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -22,10 +22,12 @@
         , roots/0
         , fields/1
         , authenticator_type/0
+        , root_type/0
+        , mechanism/1
+        , backend/1
         ]).
 
-%% only for doc generation
+roots() -> [].
-roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}].
 
 fields(_) -> [].
 
@@ -35,6 +37,7 @@ common_fields() ->
 
 enable(type) -> boolean();
 enable(default) -> true;
+enable(desc) -> "Set to <code>false</code> to disable this auth provider";
 enable(_) -> undefined.
 
 authenticator_type() ->
@@ -42,3 +45,18 @@ authenticator_type() ->
 
 config_refs(Modules) ->
     lists:append([Module:refs() || Module <- Modules]).
+
+%% authn is a core functionality however implemented outside fo emqx app
+%% in emqx_schema, 'authentication' is a map() type which is to allow
+%% EMQ X more plugable.
+root_type() ->
+    T = authenticator_type(),
+    hoconsc:union([T, hoconsc:array(T)]).
+
+mechanism(Name) ->
+    hoconsc:mk(hoconsc:enum([Name]),
+               #{nullable => false}).
+
+backend(Name) ->
+    hoconsc:mk(hoconsc:enum([Name]),
+               #{nullable => false}).

apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl

@@ -83,11 +83,11 @@ mnesia(boot) ->
 
 namespace() -> "authn-scram-builtin_db".
 
-roots() -> [config].
+roots() -> [?CONF_NS].
 
-fields(config) ->
+fields(?CONF_NS) ->
-    [ {mechanism,       {enum, [scram]}}
+    [ {mechanism, emqx_authn_schema:mechanism('scram')}
-    , {backend,         {enum, ['built-in-database']}}
+    , {backend, emqx_authn_schema:backend('built-in-database')}
     , {algorithm,       fun algorithm/1}
     , {iteration_count, fun iteration_count/1}
     ] ++ emqx_authn_schema:common_fields().
@@ -105,7 +105,7 @@ iteration_count(_) -> undefined.
 %%------------------------------------------------------------------------------
 
 refs() ->
-   [hoconsc:ref(?MODULE, config)].
+   [hoconsc:ref(?MODULE, ?CONF_NS)].
 
 create(AuthenticatorID,
        #{algorithm := Algorithm,

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -43,8 +43,9 @@
 namespace() -> "authn-http".
 
 roots() ->
-    [ {config, hoconsc:mk(hoconsc:union(refs()),
+    [ {?CONF_NS,
-                          #{})}
+       hoconsc:mk(hoconsc:union(refs()),
+                  #{})}
     ].
 
 fields(get) ->
@@ -60,8 +61,8 @@ fields(post) ->
     ] ++ common_fields().
 
 common_fields() ->
-    [ {mechanism,       hoconsc:enum(['password-based'])}
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-    , {backend,         hoconsc:enum(['http'])}
+    , {backend, emqx_authn_schema:backend(http)}
     , {url,             fun url/1}
     , {body,            fun body/1}
     , {request_timeout, fun request_timeout/1}
@@ -233,9 +234,9 @@ transform_header_name(Headers) ->
               end, #{}, Headers).
 
 check_ssl_opts(Conf) ->
-    case parse_url(hocon_schema:get_value("config.url", Conf)) of
+    case parse_url(get_conf_val("url", Conf)) of
         #{scheme := https} ->
-            case hocon_schema:get_value("config.ssl.enable", Conf) of
+            case get_conf_val("ssl.enable", Conf) of
                 true -> ok;
                 false -> false
             end;
@@ -244,8 +245,8 @@ check_ssl_opts(Conf) ->
     end.
 
 check_headers(Conf) ->
-    Method = to_bin(hocon_schema:get_value("config.method", Conf)),
+    Method = to_bin(get_conf_val("method", Conf)),
-    Headers = hocon_schema:get_value("config.headers", Conf),
+    Headers = get_conf_val("headers", Conf),
     Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)).
 
 parse_url(URL) ->
@@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) ->
     B;
 to_bin(L) when is_list(L) ->
     list_to_binary(L).
+
+get_conf_val(Name, Conf) ->
+    hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf).

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -16,6 +16,7 @@
 
 -module(emqx_authn_jwt).
 
+-include("emqx_authn.hrl").
 -include_lib("typerefl/include/types.hrl").
 
 -behaviour(hocon_schema).
@@ -40,9 +41,9 @@
 namespace() -> "authn-jwt".
 
 roots() ->
-    [ {config, hoconsc:mk(hoconsc:union(refs()),
+    [ {?CONF_NS,
-                          #{}
+       hoconsc:mk(hoconsc:union(refs()),
-                         )}
+                  #{})}
     ].
 
 fields('hmac-based') ->
@@ -82,7 +83,7 @@ fields(ssl_disable) ->
     [ {enable, #{type => false}} ].
 
 common_fields() ->
-    [ {mechanism,       {enum, [jwt]}}
+    [ {mechanism, emqx_authn_schema:mechanism('jwt')}
     , {verify_claims,   fun verify_claims/1}
     ] ++ emqx_authn_schema:common_fields().
 

apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl

@@ -85,11 +85,11 @@ mnesia(boot) ->
 
 namespace() -> "authn-builtin_db".
 
-roots() -> [config].
+roots() -> [?CONF_NS].
 
-fields(config) ->
+fields(?CONF_NS) ->
-    [ {mechanism,               {enum, ['password-based']}}
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-    , {backend,                 {enum, ['built-in-database']}}
+    , {backend, emqx_authn_schema:backend('built-in-database')}
     , {user_id_type,            fun user_id_type/1}
     , {password_hash_algorithm, fun password_hash_algorithm/1}
     ] ++ emqx_authn_schema:common_fields();
@@ -104,7 +104,7 @@ fields(other_algorithms) ->
     ].
 
 user_id_type(type) -> user_id_type();
-user_id_type(default) -> username;
+user_id_type(default) -> <<"username">>;
 user_id_type(_) -> undefined.
 
 password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt),
@@ -121,7 +121,7 @@ salt_rounds(_) -> undefined.
 %%------------------------------------------------------------------------------
 
 refs() ->
-   [hoconsc:ref(?MODULE, config)].
+   [hoconsc:ref(?MODULE, ?CONF_NS)].
 
 create(AuthenticatorID,
        #{user_id_type := Type,

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -42,8 +42,8 @@
 namespace() -> "authn-mongodb".
 
 roots() ->
-    [ {config, hoconsc:mk(hoconsc:union(refs()),
+    [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()),
-                          #{})}
+                            #{})}
     ].
 
 fields(standalone) ->
@@ -56,8 +56,8 @@ fields('sharded-cluster') ->
     common_fields() ++ emqx_connector_mongo:fields(sharded).
 
 common_fields() ->
-    [ {mechanism,               {enum, ['password-based']}}
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-    , {backend,                 {enum, [mongodb]}}
+    , {backend, emqx_authn_schema:backend(mongodb)}
     , {collection,              fun collection/1}
     , {selector,                fun selector/1}
     , {password_hash_field,     fun password_hash_field/1}

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -41,11 +41,11 @@
 
 namespace() -> "authn-mysql".
 
-roots() -> [config].
+roots() -> [?CONF_NS].
 
-fields(config) ->
+fields(?CONF_NS) ->
-    [ {mechanism,               {enum, ['password-based']}}
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-    , {backend,                 {enum, [mysql]}}
+    , {backend, emqx_authn_schema:backend(mysql)}
     , {password_hash_algorithm, fun password_hash_algorithm/1}
     , {salt_position,           fun salt_position/1}
     , {query,                   fun query/1}
@@ -74,7 +74,7 @@ query_timeout(_) -> undefined.
 %%------------------------------------------------------------------------------
 
 refs() ->
-   [hoconsc:ref(?MODULE, config)].
+   [hoconsc:ref(?MODULE, ?CONF_NS)].
 
 create(_AuthenticatorID, Config) ->
     create(Config).

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -47,11 +47,11 @@
 
 namespace() -> "authn-postgresql".
 
-roots() -> [config].
+roots() -> [?CONF_NS].
 
-fields(config) ->
+fields(?CONF_NS) ->
-    [ {mechanism,               {enum, ['password-based']}}
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-    , {backend,                 {enum, [postgresql]}}
+    , {backend, emqx_authn_schema:backend(postgresql)}
     , {password_hash_algorithm, fun password_hash_algorithm/1}
     , {salt_position,           fun salt_position/1}
     , {query,                   fun query/1}
@@ -75,7 +75,7 @@ query(_) -> undefined.
 %%------------------------------------------------------------------------------
 
 refs() ->
-    [hoconsc:ref(?MODULE, config)].
+    [hoconsc:ref(?MODULE, ?CONF_NS)].
 
 create(_AuthenticatorID, Config) ->
     create(Config).

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -42,8 +42,8 @@
 namespace() -> "authn-redis".
 
 roots() ->
-    [ {config, hoconsc:mk(hoconsc:union(refs()),
+    [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()),
-                          #{})}
+                            #{})}
     ].
 
 fields(standalone) ->
@@ -56,11 +56,11 @@ fields(sentinel) ->
     common_fields() ++ emqx_connector_redis:fields(sentinel).
 
 common_fields() ->
-    [{mechanism,               {enum, ['password-based']}},
+    [ {mechanism, emqx_authn_schema:mechanism('password-based')}
-     {backend,                 {enum, [redis]}},
+    , {backend, emqx_authn_schema:backend(redis)}
-     {cmd,                     fun cmd/1},
+    , {cmd,                     fun cmd/1}
-     {password_hash_algorithm, fun password_hash_algorithm/1},
+    , {password_hash_algorithm, fun password_hash_algorithm/1}
-     {salt_position,           fun salt_position/1}
+    , {salt_position,           fun salt_position/1}
     ] ++ emqx_authn_schema:common_fields().
 
 cmd(type) -> string();

apps/emqx_authn/test/emqx_authn_api_SUITE.erl

@@ -45,11 +45,11 @@ groups() ->
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     emqx_authn_test_lib:delete_authenticators(
-      [authentication],
+      [?CONF_NS_ATOM],
       ?GLOBAL),
 
     emqx_authn_test_lib:delete_authenticators(
-      [listeners, tcp, default, authentication],
+      [listeners, tcp, default, ?CONF_NS_ATOM],
       ?TCP_DEFAULT),
 
     {atomic, ok} = mria:clear_table(emqx_authn_mnesia),
@@ -89,8 +89,8 @@ set_special_configs(_App) ->
 %%------------------------------------------------------------------------------
 
 t_invalid_listener(_) ->
-    {ok, 404, _} = request(get, uri(["listeners", "invalid", "authentication"])),
+    {ok, 404, _} = request(get, uri(["listeners", "invalid", ?CONF_NS])),
-    {ok, 404, _} = request(get, uri(["listeners", "in:valid", "authentication"])).
+    {ok, 404, _} = request(get, uri(["listeners", "in:valid", ?CONF_NS])).
 
 t_authenticators(_) ->
     test_authenticators([]).
@@ -133,86 +133,86 @@ test_authenticators(PathPrefix) ->
     ValidConfig = emqx_authn_test_lib:http_example(),
     {ok, 200, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      ValidConfig),
 
     {ok, 409, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      ValidConfig),
 
     InvalidConfig0 = ValidConfig#{method => <<"delete">>},
     {ok, 400, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      InvalidConfig0),
 
     InvalidConfig1 = ValidConfig#{method => <<"get">>,
                                   headers => #{<<"content-type">> => <<"application/json">>}},
     {ok, 400, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      InvalidConfig1),
 
     ?assertAuthenticatorsMatch(
        [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}],
-       PathPrefix ++ ["authentication"]).
+       PathPrefix ++ [?CONF_NS]).
 
 test_authenticator(PathPrefix) ->
     ValidConfig0 = emqx_authn_test_lib:http_example(),
     {ok, 200, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      ValidConfig0),
     {ok, 200, _} = request(
                      get,
-                     uri(PathPrefix ++ ["authentication", "password-based:http"])),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:http"])),
 
     {ok, 404, _} = request(
                      get,
-                     uri(PathPrefix ++ ["authentication", "password-based:redis"])),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])),
 
 
     {ok, 404, _} = request(
                      put,
-                     uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database"]),
                      emqx_authn_test_lib:built_in_database_example()),
 
     InvalidConfig0 = ValidConfig0#{method => <<"delete">>},
     {ok, 400, _} = request(
                      put,
-                     uri(PathPrefix ++ ["authentication", "password-based:http"]),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
                      InvalidConfig0),
 
     InvalidConfig1 = ValidConfig0#{method => <<"get">>,
                                   headers => #{<<"content-type">> => <<"application/json">>}},
     {ok, 400, _} = request(
                      put,
-                     uri(PathPrefix ++ ["authentication", "password-based:http"]),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
                      InvalidConfig1),
 
     ValidConfig1 = ValidConfig0#{pool_size => 9},
     {ok, 200, _} = request(
                      put,
-                     uri(PathPrefix ++ ["authentication", "password-based:http"]),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
                      ValidConfig1),
 
     {ok, 404, _} = request(
                      delete,
-                     uri(PathPrefix ++ ["authentication", "password-based:redis"])),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])),
 
     {ok, 204, _} = request(
                      delete,
-                     uri(PathPrefix ++ ["authentication", "password-based:http"])),
+                     uri(PathPrefix ++ [?CONF_NS, "password-based:http"])),
 
-    ?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]).
+    ?assertAuthenticatorsMatch([], PathPrefix ++ [?CONF_NS]).
 
 test_authenticator_users(PathPrefix) ->
-    UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]),
+    UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]),
 
     {ok, 200, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      emqx_authn_test_lib:built_in_database_example()),
 
     InvalidUsers = [
@@ -263,11 +263,11 @@ test_authenticator_users(PathPrefix) ->
        lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])).
 
 test_authenticator_user(PathPrefix) ->
-    UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]),
+    UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]),
 
     {ok, 200, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      emqx_authn_test_lib:built_in_database_example()),
 
     User = #{user_id => <<"u1">>, password => <<"p1">>},
@@ -311,7 +311,7 @@ test_authenticator_move(PathPrefix) ->
       fun(Conf) ->
               {ok, 200, _} = request(
                                post,
-                               uri(PathPrefix ++ ["authentication"]),
+                               uri(PathPrefix ++ [?CONF_NS]),
                                Conf)
       end,
       AuthenticatorConfs),
@@ -322,40 +322,40 @@ test_authenticator_move(PathPrefix) ->
         #{<<"mechanism">> := <<"jwt">>},
         #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
        ],
-       PathPrefix ++ ["authentication"]),
+       PathPrefix ++ [?CONF_NS]),
 
     % Invalid moves
 
     {ok, 400, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"up">>}),
 
     {ok, 400, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{}),
 
     {ok, 404, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"before:invalid">>}),
 
     {ok, 404, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"before:password-based:redis">>}),
 
     {ok, 404, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"before:password-based:redis">>}),
 
     % Valid moves
 
     {ok, 204, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"top">>}),
 
     ?assertAuthenticatorsMatch(
@@ -364,11 +364,11 @@ test_authenticator_move(PathPrefix) ->
         #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>},
         #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
        ],
-       PathPrefix ++ ["authentication"]),
+       PathPrefix ++ [?CONF_NS]),
 
     {ok, 204, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"bottom">>}),
 
     ?assertAuthenticatorsMatch(
@@ -377,11 +377,11 @@ test_authenticator_move(PathPrefix) ->
         #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>},
         #{<<"mechanism">> := <<"jwt">>}
        ],
-       PathPrefix ++ ["authentication"]),
+       PathPrefix ++ [?CONF_NS]),
 
     {ok, 204, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication", "jwt", "move"]),
+                     uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
                      #{position => <<"before:password-based:built-in-database">>}),
 
     ?assertAuthenticatorsMatch(
@@ -390,17 +390,17 @@ test_authenticator_move(PathPrefix) ->
         #{<<"mechanism">> := <<"jwt">>},
         #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
        ],
-       PathPrefix ++ ["authentication"]).
+       PathPrefix ++ [?CONF_NS]).
 
 test_authenticator_import_users(PathPrefix) ->
     ImportUri = uri(
                   PathPrefix ++
-                  ["authentication", "password-based:built-in-database", "import_users"]),
+                  [?CONF_NS, "password-based:built-in-database", "import_users"]),
 
 
     {ok, 200, _} = request(
                      post,
-                     uri(PathPrefix ++ ["authentication"]),
+                     uri(PathPrefix ++ [?CONF_NS]),
                      emqx_authn_test_lib:built_in_database_example()),
 
     {ok, 400, _} = request(post, ImportUri, #{}),

apps/emqx_authn/test/emqx_authn_http_SUITE.erl

@@ -24,7 +24,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
 
--define(PATH, [authentication]).
+-define(PATH, [?CONF_NS_ATOM]).
 
 -define(HTTP_PORT, 33333).
 -define(HTTP_PATH, "/auth").

apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl

@@ -49,6 +49,8 @@ end_per_testcase(_Case, Config) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
+-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}).
+
 t_check_schema(_Config) ->
     ConfigOk = #{
         <<"mechanism">> => <<"password-based">>,
@@ -60,7 +62,7 @@ t_check_schema(_Config) ->
         }
     },
 
-    hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}),
+    hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)),
 
     ConfigNotOk = #{
         <<"mechanism">> => <<"password-based">>,
@@ -74,7 +76,7 @@ t_check_schema(_Config) ->
     ?assertException(
         throw,
         {emqx_authn_mnesia, _},
-        hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})).
+        hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))).
 
 t_create(_) ->
     Config0 = config(),

apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl

@@ -31,7 +31,7 @@ groups() ->
 
 init_per_suite(Config) ->
     ok = emqx_common_test_helpers:start_apps(
-           [emqx_conf, emqx_authz],
+           [emqx_connector, emqx_conf, emqx_authz],
            fun set_special_configs/1
           ),
     Config.

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -33,7 +33,7 @@
     catch
         error:{invalid_bridge_id, Id0} ->
             {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary,
-                ". Bridge Ids must be of format <bridge_type>:<name>">>}}
+                ". Bridge ID must be of format 'bridge_type:name'">>}}
     end).
 
 -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX),

apps/emqx_bridge/src/emqx_bridge_schema.erl

@@ -108,7 +108,8 @@ connector_name() ->
            #{ nullable => false
             , desc =>"""
 The connector name to be used for this bridge.
-Connectors are configured by 'connectors.<type>.<name>
+Connectors are configured as 'connectors.type.name',
+for example 'connectors.http.mybridge'.
 """
             })}.
 

apps/emqx_conf/etc/emqx_conf.md

@@ -0,0 +1,192 @@
+EMQ X configuration file is in [HOCON](https://github.com/emqx/hocon) format.
+HOCON, or Human-Optimized Config Object Notation is a format for human-readable data,
+and a superset of JSON.
+
+## Syntax
+
+In config file the values can be notated as JSON like ojbects, such as
+```
+node {
+    name = "emqx@127.0.0.1"
+    cookie = "mysecret"
+}
+```
+
+Another equivalent representation is flat, suh as
+
+```
+node.name="127.0.0.1"
+node.cookie="mysecret"
+```
+
+This flat format is almost backward compatible with EMQ X's config file format
+in 4.x series (the so called 'cuttlefish' format).
+
+It is 'almost' compabile because the often HOCON requires strings to be quoted,
+while cuttlefish treats all characters to the right of the `=` mark as the value.
+
+e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"`
+
+Strings without special characters in them can be unquoted in HOCON too,
+e.g. `foo`, `foo_bar`, `foo_bar_1`:
+
+For more HOCON syntax, pelase refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md)
+
+## Schema
+
+To make the HOCON objects type-safe, EMQ X introduded a schema for it.
+The schema defines data types, and data fields' names and metadata for config value validation
+and more. In fact, this config document itself is generated from schema metadata.
+
+### Complex Data Types
+
+There are 4 complex data types in EMQ X's HOCON config:
+
+1. Struct: Named using an unquoted string, followed by a pre-defined list of fields,
+   fields can not start with a number, and are only allowed to use
+   lowercase letters and underscores as word separater.
+1. Map: Map is like Struct, however the fields are not pre-defined.
+   1-based index number can also be used as map keys for an alternative
+   representation of an Array.
+1. Union: `MemberType1 | MemberType2 | ...`
+1. Array: `[ElementType]`
+
+### Primitive Data Types
+
+Complex types define data 'boxes' wich may contain other complex data
+or primitive values.
+There are quite some different primitive types, to name a fiew:
+
+* `atom()`
+* `boolean()`
+* `string()`
+* `integer()`
+* `float()`
+* `number()`
+* `binary()` # another format of string()
+* `emqx_schema:duration()` # time duration, another format of integer()
+* ...
+
+The primitive types are mostly self-describing, some are built-in, such
+as `atom()`, some are defiend in EMQ X modules, such as `emqx_schema:duration()`.
+
+### Config Paths
+
+If we consider the whole EMQ X config as a tree,
+to reference a primitive value, we can use a dot-separated names form string for
+the path from the tree-root (always a Struct) down to the primitive values at tree-leaves.
+
+Each segment of the dotted string is a Struct filed name or Map key.
+For Array elements, 1-based index is used.
+
+below are some examples
+
+```
+node.name="emqx.127.0.0.1"
+zone.zone1.max_packet_size="10M"
+authentication.1.enable=true
+```
+
+### Environment varialbes
+
+Environment variables can be used to define or override config values.
+
+Due to the fact that dots (`.`) are not allowed in environment variables, dots are
+replaced with double-underscores (`__`).
+
+And a the `EMQX_` prefix is used as the namespace.
+
+For example `node.name` can be represented as `EMQX_NODE__NAME`
+
+Environment varialbe values are parsed as hocon values, this allows users
+to even set complex values from environment variables.
+
+For example, this environment variable sets an array value.
+
+```
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+```
+
+Unknown environment variables are logged as a `warning` level log, for example:
+
+```
+[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"]
+```
+
+because the field name is `enable`, not `enabled`.
+
+<strong>NOTE:</strong> Unknown root keys are however silently discarded.
+
+### Config overlay
+
+HOCON values are overlayed, earlier defined values are at layers closer to the bottom.
+The overall order of the overlay rules from bottom up are:
+
+1. `emqx.conf` the base config file
+1. `EMQX_` prfixed environment variables
+1. Cluster override file, the path of which is configured as `cluster_override_conf_file` in the lower layers
+1. Local override file, the path of which is configured as `local_override_conf_file` in the lower layers
+
+Below are the rules of config value overlay.
+
+#### Struct Fileds
+
+Later config values overwrites earlier values.
+For example, in below config, the last line `debug` overwrites `errro` for
+console log handler's `level` config, but leaving `enable` unchanged.
+```
+log {
+    console_handler{
+        enable=true,
+        level=error
+    }
+}
+
+## ... more configs ...
+
+log.console_handler.level=debug
+```
+
+#### Map Values
+
+Maps are like structs, only the files are user-defined rather than
+the config schema. For instance, `zone1` in the exampele below.
+
+```
+zone {
+    zone1 {
+        mqtt.max_packet_size = 1M
+    }
+}
+
+## The maximum packet size can be defined as above,
+## then overriden as below
+
+zone.zone1.mqtt.max_packet_size = 10M
+```
+
+#### Array Elements
+
+Arrays in EMQ X config have two different representations
+
+* list, such as: `[1, 2, 3]`
+* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}`
+
+Dot-separated paths with number in it are parsed to indexed-maps
+e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}`
+
+Indexed-map arrays can be used to override list arrays:
+
+```
+authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}]
+# we can disable this authentication provider with:
+authentication.1.enable=false
+```
+However, list arrays do not get recursively merged into indexed-map arrays.
+e.g.
+
+```
+authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}]
+## below value will replace the whole array, but not to override just one field.
+authentication=[{enable=true}]
+```

apps/emqx_conf/src/emqx_conf.erl

@@ -24,6 +24,7 @@
 -export([update/3, update/4]).
 -export([remove/2, remove/3]).
 -export([reset/2, reset/3]).
+-export([gen_doc/1]).
 
 %% for rpc
 -export([get_node_and_config/1]).
@@ -123,6 +124,16 @@ reset(Node, KeyPath, Opts) when Node =:= node() ->
 reset(Node, KeyPath, Opts) ->
     rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]).
 
+-spec gen_doc(file:name_all()) -> ok.
+gen_doc(File) ->
+    Version = emqx_release:version(),
+    Title = "# EMQ X " ++ Version ++ " Configuration",
+    BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]),
+    {ok, Body} = file:read_file(BodyFile),
+    Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title,
+                                                   body => Body}),
+    file:write_file(File, Doc).
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -24,6 +24,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
 
 -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all.
 -type file() :: string().
@@ -62,8 +63,12 @@
 namespace() -> undefined.
 
 roots() ->
-    %% authorization configs are merged in THIS schema's "authorization" fields
+    PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY,
-    lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++
+    case persistent_term:get(PtKey, undefined) of
+        undefined -> persistent_term:put(PtKey, emqx_authn_schema);
+        _ -> ok
+    end,
+    emqx_schema_high_prio_roots() ++
     [ {"node",
        sc(hoconsc:ref("node"),
           #{ desc => "Node name, cookie, config & data directories "
@@ -87,20 +92,6 @@ roots() ->
                      "should work, but in case you need to do performance "
                      "fine-turning or experiment a bit, this is where to look."
            })}
-    , {"authorization",
-       sc(hoconsc:ref("authorization"),
-          #{ desc => """
-Authorization a.k.a ACL.<br>
-In EMQ X, MQTT client access control is extremly flexible.<br>
-An out of the box set of authorization data sources are supported.
-For example,<br>
-'file' source is to support concise and yet generic ACL rules in a file;<br>
-'built-in-database' source can be used to store per-client customisable rule sets,
-natively in the EMQ X node;<br>
-'http' source to make EMQ X call an external HTTP API to make the decision;<br>
-'postgresql' etc. to look up clients or rules from external databases;<br>
-"""
-           })}
     , {"db",
        sc(ref("db"),
           #{ desc => "Settings of the embedded database."
@@ -251,14 +242,12 @@ fields("node") ->
     [ {"name",
        sc(string(),
           #{ default => "emqx@127.0.0.1"
-           , override_env => "EMQX_NODE_NAME"
            })}
     , {"cookie",
        sc(string(),
           #{ mapping => "vm_args.-setcookie",
              default => "emqxsecretcookie",
-             sensitive => true,
+             sensitive => true
-             override_env => "EMQX_NODE_COOKIE"
            })}
     , {"data_dir",
        sc(string(),
@@ -845,3 +834,22 @@ ensure_list(V) ->
 
 roots(Module) ->
     lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)).
+
+%% Like authentication schema, authorization schema is incomplete in emqx_schema
+%% module, this function replaces the root filed "authorization" with a new schema
+emqx_schema_high_prio_roots() ->
+    Roots = emqx_schema:roots(high),
+    Authz = {"authorization",
+             sc(hoconsc:ref("authorization"),
+             #{ desc => """
+Authorization a.k.a ACL.<br>
+In EMQ X, MQTT client access control is extremly flexible.<br>
+An out of the box set of authorization data sources are supported.
+For example,<br>
+'file' source is to support concise and yet generic ACL rules in a file;<br>
+'built-in-database' source can be used to store per-client customisable rule sets,
+natively in the EMQ X node;<br>
+'http' source to make EMQ X call an external HTTP API to make the decision;<br>
+'postgresql' etc. to look up clients or rules from external databases;<br>
+""" })},
+    lists:keyreplace("authorization", 1, Roots, Authz).

apps/emqx_connector/src/emqx_connector_api.erl

@@ -38,7 +38,7 @@
     catch
         error:{invalid_bridge_id, Id0} ->
             {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary,
-                ". Bridge Ids must be of format <bridge_type>:<name>">>}}
+                ". Bridge ID must be of format 'bridge_type:name'">>}}
     end).
 
 namespace() -> "connector".
@@ -74,7 +74,7 @@ schema("/connectors_test") ->
         post => #{
             tags => [<<"connectors">>],
             description => <<"Test creating a new connector by given Id <br>"
-                             "The Id must be of format <type>:<name>">>,
+                             "The ID must be of format 'type:name'">>,
             summary => <<"Test creating connector">>,
             requestBody => connector_test_info(),
             responses => #{
@@ -98,7 +98,7 @@ schema("/connectors") ->
         post => #{
             tags => [<<"connectors">>],
             description => <<"Create a new connector by given Id <br>"
-                             "The Id must be of format <type>:<name>">>,
+                             "The ID must be of format 'type:name'">>,
             summary => <<"Create connector">>,
             requestBody => connector_info(),
             responses => #{

apps/emqx_connector/src/emqx_connector_schema.erl

@@ -11,6 +11,7 @@
 
 roots() -> ["connectors"].
 
+fields(connectors) -> fields("connectors");
 fields("connectors") ->
     [ {mqtt,
        sc(hoconsc:map(name,

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -182,12 +182,12 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc,
     Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
     case hocon_schema:field_schema(Type, in) of
         path ->
-            Option = #{atom_key => true, override_env => false},
+            Option = #{atom_key => true},
             NewBindings = hocon_schema:check_plain(Schema, Bindings, Option),
             NewBindingsAcc = maps:merge(BindingsAcc, NewBindings),
             check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc);
         query ->
-            Option = #{override_env => false},
+            Option = #{},
             NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option),
             NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
             check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc)
@@ -201,7 +201,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
             _ -> Type0
         end,
     NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]},
-    Option = #{override_env => false, nullable => true},
+    Option = #{nullable => true},
     #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option),
     NewBody;
 %% TODO not support nest object check yet, please use ref!
@@ -214,7 +214,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
 check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
     lists:foldl(fun({Name, Type}, Acc) ->
         Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
-        maps:merge(Acc, CheckFun(Schema, Body, #{override_env => false}))
+        maps:merge(Acc, CheckFun(Schema, Body, #{}))
                 end, #{}, Spec).
 
 %% tags, description, summary, security, deprecated

apps/emqx_gateway/src/coap/emqx_coap_channel.erl

@@ -18,9 +18,6 @@
 
 -behaviour(emqx_gateway_channel).
 
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-
 %% API
 -export([ info/1
         , info/2
@@ -44,6 +41,12 @@
 
 -export_type([channel/0]).
 
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
+
+-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
+
 -record(channel, {
                   %% Context
                   ctx           :: emqx_gateway_ctx:context(),
@@ -283,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) ->
             %% udp connection baseon the clientid
             call_session(handle_request, Msg, Channel);
         _ ->
-            case emqx_conf:get([gateway, coap, authentication], undefined) of
+            case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of
                 undefined ->
                     call_session(handle_request, Msg, Channel);
                 _ ->

apps/emqx_gateway/src/emqx_gateway_api.erl

@@ -17,6 +17,7 @@
 -module(emqx_gateway_api).
 
 -include_lib("emqx/include/emqx_placeholder.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
 
 -behaviour(minirest_api).
 
@@ -243,7 +244,7 @@ schema_gateway_overview_list() ->
 %%
 %% NOTE: It is a temporary measure to generate swagger-schema
 -define(COAP_GATEWAY_CONFS,
-#{<<"authentication">> =>
+#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY =>
       #{<<"mechanism">> => <<"password-based">>,
         <<"name">> => <<"authenticator1">>,
         <<"server_type">> => <<"built-in-database">>,
@@ -331,7 +332,7 @@ schema_gateway_overview_list() ->
 ).
 
 -define(STOMP_GATEWAY_CONFS,
-#{<<"authentication">> =>
+#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY =>
       #{<<"mechanism">> => <<"password-based">>,
         <<"name">> => <<"authenticator1">>,
         <<"server_type">> => <<"built-in-database">>,

apps/emqx_gateway/src/emqx_gateway_conf.erl

@@ -17,8 +17,6 @@
 %% @doc The gateway configuration management module
 -module(emqx_gateway_conf).
 
--include_lib("emqx/include/logger.hrl").
-
 %% Load/Unload
 -export([ load/0
         , unload/0
@@ -56,6 +54,10 @@
         , post_config_update/5
         ]).
 
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
+-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY).
+
 -type atom_or_bin() :: atom() | binary().
 -type ok_or_err() :: ok_or_err().
 -type listener_ref() :: {ListenerType :: atom_or_bin(),
@@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) ->
 
 -spec update_gateway(atom_or_bin(), map()) -> ok_or_err().
 update_gateway(GwName, Conf0) ->
-    Conf = maps:without([listeners, authentication,
+    Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM],
-                         <<"listeners">>, <<"authentication">>], Conf0),
+    Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0],
+    Conf = maps:without(Exclude0 ++ Exclude1, Conf0),
     update({?FUNCTION_NAME, bin(GwName), Conf}).
 
 %% FIXME: delete cert files ??
@@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
         undefined ->
             {error, not_found};
         _ ->
-            NConf = maps:without([<<"listeners">>,
+            NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
-                                  <<"authentication">>], Conf),
             {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})}
     end;
 pre_config_update(_, {unload_gateway, GwName}, RawConf) ->
@@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) ->
 
 pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
-           [GwName, <<"authentication">>], RawConf, undefined) of
+           [GwName, ?AUTHN_BIN], RawConf, undefined) of
         undefined ->
             {ok, emqx_map_lib:deep_merge(
                    RawConf,
-                   #{GwName => #{<<"authentication">> => Conf}})};
+                   #{GwName => #{?AUTHN_BIN => Conf}})};
         _ ->
             {error, already_exist}
     end;
@@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
         undefined ->
             {error, not_found};
         Listener ->
-            case maps:get(<<"authentication">>, Listener, undefined) of
+            case maps:get(?AUTHN_BIN, Listener, undefined) of
                 undefined ->
-                    NListener = maps:put(<<"authentication">>, Conf, Listener),
+                    NListener = maps:put(?AUTHN_BIN, Conf, Listener),
                     NGateway = #{GwName =>
                                  #{<<"listeners">> =>
                                    #{LType => #{LName => NListener}}}},
@@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
     end;
 pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
-           [GwName, <<"authentication">>], RawConf, undefined) of
+           [GwName, ?AUTHN_BIN], RawConf, undefined) of
         undefined ->
             {error, not_found};
         _ ->
             {ok, emqx_map_lib:deep_merge(
                    RawConf,
-                   #{GwName => #{<<"authentication">> => Conf}})}
+                   #{GwName => #{?AUTHN_BIN => Conf}})}
     end;
 pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
@@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
         undefined ->
             {error, not_found};
         Listener ->
-            case maps:get(<<"authentication">>, Listener, undefined) of
+            case maps:get(?AUTHN_BIN, Listener, undefined) of
                 undefined ->
                     {error, not_found};
                 Auth ->
                     NListener = maps:put(
-                                  <<"authentication">>,
+                                  ?AUTHN_BIN,
                                   emqx_map_lib:deep_merge(Auth, Conf),
                                   Listener
                                  ),
@@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
     end;
 pre_config_update(_, {remove_authn, GwName}, RawConf) ->
     {ok, emqx_map_lib:deep_remove(
-           [GwName, <<"authentication">>], RawConf)};
+           [GwName, ?AUTHN_BIN], RawConf)};
 pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
-    Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>],
+    Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN],
     {ok, emqx_map_lib:deep_remove(Path, RawConf)};
 
 pre_config_update(_, UnknownReq, _RawConf) ->

apps/emqx_gateway/src/emqx_gateway_http.erl

@@ -19,6 +19,9 @@
 
 -include("include/emqx_gateway.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
+
+-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
 
 %% Mgmt APIs - gateway
 -export([ gateways/1
@@ -166,7 +169,7 @@ remove_listener(ListenerId) ->
 -spec authn(gateway_name()) -> map().
 authn(GwName) ->
     %% XXX: Need append chain-nanme, authenticator-id?
-    Path = [gateway, GwName, authentication],
+    Path = [gateway, GwName, ?AUTHN],
     ChainName = emqx_gateway_utils:global_chain(GwName),
     wrap_chain_name(
       ChainName,
@@ -176,7 +179,7 @@ authn(GwName) ->
 -spec authn(gateway_name(), binary()) -> map().
 authn(GwName, ListenerId) ->
     {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
-    Path = [gateway, GwName, listeners, Type, Name, authentication],
+    Path = [gateway, GwName, listeners, Type, Name, ?AUTHN],
     ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name),
     wrap_chain_name(
       ChainName,

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -24,6 +24,7 @@
 -dialyzer(no_unused).
 -dialyzer(no_fail_call).
 
+-include_lib("emqx/include/emqx_authentication.hrl").
 -include_lib("typerefl/include/types.hrl").
 
 -type ip_port() :: tuple().
@@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW"
            , desc =>
 "The Pre-defined topic ids and topic names.<br>
 A 'pre-defined' topic id is a topic id whose mapping to a topic name
-is known in advance by both the client’s application and the gateway"
+is known in advance by both the client's application and the gateway"
            })}
     , {listeners, sc(ref(udp_listeners))}
     ] ++ gateway_common_options();
@@ -407,30 +408,14 @@ fields(dtls_opts) ->
          , ciphers => dtls_all_available
          }, false).
 
-authentication() ->
+authentication_schema() ->
-    sc(hoconsc:union(
+    sc(emqx_authn_schema:authenticator_type(),
-         [ hoconsc:ref(emqx_authn_mnesia, config)
+       #{ nullable => {true, recursively}
-         , hoconsc:ref(emqx_authn_mysql, config)
+        , desc =>
-         , hoconsc:ref(emqx_authn_pgsql, config)
-         , hoconsc:ref(emqx_authn_mongodb, standalone)
-         , hoconsc:ref(emqx_authn_mongodb, 'replica-set')
-         , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster')
-         , hoconsc:ref(emqx_authn_redis, standalone)
-         , hoconsc:ref(emqx_authn_redis, cluster)
-         , hoconsc:ref(emqx_authn_redis, sentinel)
-         , hoconsc:ref(emqx_authn_http, get)
-         , hoconsc:ref(emqx_authn_http, post)
-         , hoconsc:ref(emqx_authn_jwt, 'hmac-based')
-         , hoconsc:ref(emqx_authn_jwt, 'public-key')
-         , hoconsc:ref(emqx_authn_jwt, 'jwks')
-         , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config)
-         ]),
-         #{ nullable => {true, recursively}
-          , desc =>
 """Default authentication configs for all of the gateway listeners.<br>
 For per-listener overrides see <code>authentication</code>
 in listener configs"""
-          }).
+        }).
 
 gateway_common_options() ->
     [ {enable,
@@ -464,7 +449,7 @@ it has two purposes:
        sc(ref(clientinfo_override),
           #{ desc => ""
            })}
-    , {authentication,  authentication()}
+    , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,  authentication_schema()}
     ].
 
 common_listener_opts() ->
@@ -483,7 +468,7 @@ common_listener_opts() ->
        sc(integer(),
           #{ default => 1000
            })}
-    , {authentication,  authentication()}
+    , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,  authentication_schema()}
     , {mountpoint,
        sc(binary(),
           #{ default => undefined

apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl

@@ -28,6 +28,8 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
+%% this parses to #{}, will not cause config cleanup
+%% so we will need call emqx_config:erase
 -define(CONF_DEFAULT, <<"
 gateway {}
 ">>).
@@ -39,6 +41,7 @@ gateway {}
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Conf) ->
+    emqx_config:erase(gateway),
     emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
     emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
     Conf.

apps/emqx_modules/test/emqx_rewrite_SUITE.erl

@@ -170,9 +170,9 @@ t_update_re_failed(_Config) ->
         {error,
             {emqx_modules_schema,
                 [{validation_error,
-                    #{array_index => 1,path => "rewrite.re",
+                    #{path => "rewrite.1.re",
-                        reason => {<<"*^test/*">>,{"nothing to repeat",0}},
+                      reason => {<<"*^test/*">>,{"nothing to repeat",0}},
-                        value => <<"*^test/*">>}}]}}},
+                      value => <<"*^test/*">>}}]}}},
     ?assertError(Error, emqx_rewrite:update(Rules)),
     ok.
 

apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl

@@ -56,7 +56,7 @@ A list of outputs of the rule.<br>
 An output can be a string that refers to the channel Id of a emqx bridge, or a object
 that refers to a function.<br>
 There a some built-in functions like \"republish\" and \"console\", and we also support user
-provided functions like \"<SomeModule>:<SomeFunction>\".<br>
+provided functions like \"ModuleName:FunctionName\".<br>
 The outputs in the list is executed one by one in order.
 This means that if one of the output is executing slowly, all of the outputs comes after it will not
 be executed until it returns.<br>

bin/emqx

@@ -342,6 +342,9 @@ generate_config() {
     NOW_TIME="$(call_hocon now_time)"
 
     ## ths command populates two files: app.<time>.config and vm.<time>.args
+    ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
+    ## but does not include the cluster-override.conf and local-override.conf
+    ## meaning, certain overrides will not be mapped to app.<time>.config file
     ## disable SC2086 to allow EMQX_LICENSE_CONF_OPTION to split
     # shellcheck disable=SC2086
     call_hocon -v -t "$NOW_TIME" -I "$CONFIGS_DIR/" -s $SCHEMA_MOD -c "$RUNNER_ETC_DIR"/emqx.conf $EMQX_LICENSE_CONF_OPTION -d "$RUNNER_DATA_DIR"/configs generate
@@ -451,18 +454,23 @@ case "${COMMAND}" in
         ;;
 esac
 
+## make EMQX_NODE_COOKIE right
+if [ -n "${EMQX_NODE_NAME:-}" ]; then
+    export EMQX_NODE__NAME="${EMQX_NODE_NAME}"
+    unset EMQX_NODE_NAME
+fi
 ## Possible ways to configure emqx node name:
 ## 1. configure node.name in emqx.conf
-## 2. override with environment variable EMQX_NODE_NAME
+## 2. override with environment variable EMQX_NODE__NAME
 ## Node name is either short-name (without '@'), e.g. 'emqx'
 ## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1'
-NAME="${EMQX_NODE_NAME:-}"
+NAME="${EMQX_NODE__NAME:-}"
 if [ -z "$NAME" ]; then
     if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
         # for boot commands, inspect emqx.conf for node name
         NAME="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")"
     else
-        vm_args_file="$(latest_vm_args 'EMQX_NODE_NAME')"
+        vm_args_file="$(latest_vm_args 'EMQX_NODE__NAME')"
         NAME="$(grep -E '^-s?name' "${vm_args_file}" | awk '{print $2}')"
     fi
 fi
@@ -483,18 +491,23 @@ export ESCRIPT_NAME="$SHORT_NAME"
 
 PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
 
-COOKIE="${EMQX_NODE_COOKIE:-}"
+## make EMQX_NODE_COOKIE right
+if [ -n "${EMQX_NODE_COOKIE:-}" ]; then
+    export EMQX_NODE__COOKIE="${EMQX_NODE_COOKIE}"
+    unset EMQX_NODE_COOKIE
+fi
+COOKIE="${EMQX_NODE__COOKIE:-}"
 if [ -z "$COOKIE" ]; then
     if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
         COOKIE="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")"
     else
-        vm_args_file="$(latest_vm_args 'EMQX_NODE_COOKIE')"
+        vm_args_file="$(latest_vm_args 'EMQX_NODE__COOKIE')"
         COOKIE="$(grep -E '^-setcookie' "${vm_args_file}" | awk '{print $2}')"
     fi
 fi
 
 if [ -z "$COOKIE" ]; then
-    die "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE_COOKIE"
+    die "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
 fi
 
 cd "$ROOTDIR"

build

@@ -51,16 +51,17 @@ log() {
     echo "===< $msg"
 }
 
-docgen() {
+make_doc() {
     local libs_dir1 libs_dir2
     libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
     libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
 
-    local conf_doc_markdown
+    local conf_doc_md
-    conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md"
+    # TODO render md as html
-    echo "===< Generating config document $conf_doc_markdown"
+    conf_doc_md="$(pwd)/_build/${PROFILE}/lib/emqx_dashboard/priv/config.md"
+    echo "===< Generating config document $conf_doc_md"
     # shellcheck disable=SC2086
-    erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema)), halt(0)."
+    erl -noshell -pa $libs_dir1 $libs_dir2 -eval "ok = emqx_conf:gen_doc(\"${conf_doc_md}\"), halt(0)."
 }
 
 make_rel() {
@@ -70,7 +71,6 @@ make_rel() {
         echo "gpb should not be included in the release"
         exit 1
     fi
-    docgen
 }
 
 ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
@@ -205,6 +205,9 @@ make_docker_testing() {
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
+    doc)
+        make_doc
+        ;;
     rel)
         make_rel
         ;;

rebar.config

@@ -64,7 +64,7 @@
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}}

rebar.config.erl

@@ -153,36 +153,42 @@ profiles() ->
        , {relx, relx(Vsn, cloud, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
+       , {post_hooks, [{compile, "./build emqx doc"}]}
        ]}
     , {'emqx-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
+       , {post_hooks, [{compile, "./build emqx-pkg doc"}]}
        ]}
     , {'emqx-enterprise',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, bin, ee)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ee)}
+       , {post_hooks, [{compile, "./build emqx-enterprise doc"}]}
        ]}
     , {'emqx-enterprise-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, pkg, ee)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ee)}
+       , {post_hooks, [{compile, "./build emqx-enterprise-pkg doc"}]}
        ]}
     , {'emqx-edge',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, edge, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
+       , {post_hooks, [{compile, "./build emqx-edge doc"}]}
        ]}
     , {'emqx-edge-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, edge, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
+       , {post_hooks, [{compile, "./build emqx-edge-pkg doc"}]}
        ]}
     , {check,
        [ {erl_opts, common_compile_opts()}