feat: configuration priority ENV > emqx.conf > API

apps/emqx/src/config/emqx_config_logger.erl

@@ -32,25 +32,15 @@ remove_handler() ->
     ok = emqx_config_handler:remove_handler(?LOG),
     ok.
 
-%% refresh logger config when booting, the override config may have changed after node start.
+%% refresh logger config when booting, the cluster config may have changed after node start.
 %% Kernel's app env is confirmed before the node starts,
-%% but we only copy cluster-override.conf from other node after this node starts,
+%% but we only copy cluster.conf from other node after this node starts,
 %% so we need to refresh the logger config after this node starts.
-%% It will not affect the logger config when cluster-override.conf is unchanged.
+%% It will not affect the logger config when cluster.conf is unchanged.
 refresh_config() ->
-    Overrides = emqx_config:read_override_confs(),
-    refresh_config(Overrides).
-
-refresh_config(#{<<"log">> := _}) ->
     %% read the checked config
     LogConfig = emqx:get_config(?LOG, undefined),
-    Conf = #{log => LogConfig},
+    do_refresh_config(#{log => LogConfig}).
-    ok = do_refresh_config(Conf);
-refresh_config(_) ->
-    %% No config override found for 'log', do nothing
-    %% because the 'kernel' app should already be configured
-    %% from the base configs. i.e. emqx.conf + env vars
-    ok.
 
 %% this call is shared between initial config refresh at boot
 %% and dynamic config update from HTTP API
@@ -61,10 +51,9 @@ do_refresh_config(Conf) ->
     ok = maybe_update_log_level(Level),
     ok.
 
+%% always refresh config when the override config is changed
 post_config_update(?LOG, _Req, NewConf, _OldConf, _AppEnvs) ->
-    ok = do_refresh_config(#{log => NewConf});
+    do_refresh_config(#{log => NewConf}).
-post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
-    ok.
 
 maybe_update_log_level(NewLevel) ->
     OldLevel = emqx_logger:get_primary_log_level(),

apps/emqx/src/emqx_config.erl

@@ -24,17 +24,19 @@
     init_load/2,
     init_load/3,
     read_override_conf/1,
-    read_override_confs/0,
     delete_override_conf_files/0,
     check_config/2,
     fill_defaults/1,
     fill_defaults/2,
     fill_defaults/3,
-    save_configs/5,
+    save_configs/6,
     save_to_app_env/1,
     save_to_config_map/2,
     save_to_override_conf/2
 ]).
+-export([raw_conf_with_default/4]).
+-export([remove_default_conf/2]).
+-export([merge_envs/2]).
 
 -export([
     get_root/1,
@@ -329,23 +331,15 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) ->
     init_load(SchemaMod, parse_hocon(Conf), Opts);
 init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) ->
     ok = save_schema_mod_and_names(SchemaMod),
+    RootNames = get_root_names(),
     %% Merge environment variable overrides on top
     RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
-    Overrides = read_override_confs(),
+    RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts),
-    RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides),
-    RootNames = get_root_names(),
-    RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts),
     %% check configs against the schema
     {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}),
     save_to_app_env(AppEnvs),
     ok = save_to_config_map(CheckedConf, RawConfAll).
 
-%% @doc Read merged cluster + local overrides.
-read_override_confs() ->
-    ClusterOverrides = read_override_conf(#{override_to => cluster}),
-    LocalOverrides = read_override_conf(#{override_to => local}),
-    hocon:deep_merge(ClusterOverrides, LocalOverrides).
-
 %% keep the raw and non-raw conf has the same keys to make update raw conf easier.
 raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) ->
     Fun = fun(Name, Acc) ->
@@ -375,6 +369,9 @@ schema_default(Schema) ->
     end.
 
 parse_hocon(Conf) ->
+    %% merge cluster-override.conf to local-override.conf
+    %% cluster-override.conf is deprecated, now is cluster.conf
+    merge_deprecated_cluster_override_to_local_override(),
     IncDirs = include_dirs(),
     case do_parse_hocon(Conf, IncDirs) of
         {ok, HoconMap} ->
@@ -393,8 +390,12 @@ parse_hocon(Conf) ->
 do_parse_hocon(Conf, IncDirs) ->
     Opts = #{format => map, include_dirs => IncDirs},
     case is_binary(Conf) of
-        true -> hocon:binary(Conf, Opts);
+        true ->
-        false -> hocon:files(Conf, Opts)
+            hocon:binary(Conf, Opts);
+        false ->
+            LocalFile = override_conf_file(#{override_to => local}),
+            ClusterFile = override_conf_file(#{override_to => cluster}),
+            hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts)
     end.
 
 include_dirs() ->
@@ -483,6 +484,13 @@ read_override_conf(#{} = Opts) ->
     File = override_conf_file(Opts),
     load_hocon_file(File, map).
 
+read_deprecated_override_conf() ->
+    ClusterFile = override_conf_file(#{override_to => cluster}),
+    DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"),
+    Conf = load_hocon_file(DeprecatedFile, map),
+    _ = file:delete(DeprecatedFile),
+    Conf.
+
 override_conf_file(Opts) when is_map(Opts) ->
     Key =
         case maps:get(override_to, Opts, cluster) of
@@ -522,14 +530,54 @@ get_schema_mod(RootName) ->
 get_root_names() ->
     maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})).
 
--spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok.
+-spec save_configs(
-save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) ->
+    emqx_map_lib:config_key_path(), app_envs(), config(), raw_config(), raw_config(), update_opts()
-    %% We first try to save to override.conf, because saving to files is more error prone
+) -> ok.
+
+save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) ->
+    Default = init_default(Paths0),
+    OverrideConf1 = remove_default_conf(OverrideConf, Default),
+    %% We first try to save to files, because saving to files is more error prone
     %% than saving into memory.
-    ok = save_to_override_conf(OverrideConf, Opts),
+    ok = save_to_override_conf(OverrideConf1, Opts),
     save_to_app_env(AppEnvs),
     save_to_config_map(Conf, RawConf).
 
+init_default(Paths0) ->
+    [Root | _] = [bin(Key) || Key <- Paths0],
+    SchemaMod = get_schema_mod(Root),
+    {_, {_, Schema}} = lists:keyfind(Root, 1, hocon_schema:roots(SchemaMod)),
+    fill_defaults(#{Root => schema_default(Schema)}).
+
+remove_default_conf(undefined, _) ->
+    undefined;
+remove_default_conf(Conf, DefaultConf) when is_map(Conf) andalso is_map(DefaultConf) ->
+    maps:fold(
+        fun(Key, Value, Acc) ->
+            case maps:find(Key, DefaultConf) of
+                {ok, DefaultValue} ->
+                    remove_default_conf(Value, DefaultValue, Key, Acc);
+                error ->
+                    Acc
+            end
+        end,
+        Conf,
+        Conf
+    ).
+
+remove_default_conf(Value, Value, Key, Conf) ->
+    maps:remove(Key, Conf);
+remove_default_conf(Value = #{}, DefaultValue = #{}, Key, Conf) ->
+    case remove_default_conf(Value, DefaultValue) of
+        SubValue when SubValue =:= #{} -> maps:remove(Key, Conf);
+        SubValue -> maps:put(Key, SubValue, Conf)
+    end;
+remove_default_conf(Value, DefaultValue, Key, Conf) ->
+    case try_bin(DefaultValue) =:= try_bin(Value) of
+        true -> maps:remove(Key, Conf);
+        false -> Conf
+    end.
+
 %% we ignore kernel app env,
 %% because the old app env will be used in emqx_config_logger:post_config_update/5
 -define(IGNORE_APPS, [kernel]).
@@ -678,6 +726,16 @@ atom(Str) when is_list(Str) ->
 atom(Atom) when is_atom(Atom) ->
     Atom.
 
+try_bin(Bin) when is_binary(Bin) -> Bin;
+try_bin([Bin | _] = List) when is_binary(Bin) -> List;
+try_bin([Atom | _] = List) when is_atom(Atom) -> [atom_to_binary(A) || A <- List];
+try_bin([Map | _] = Maps) when is_map(Map) -> Maps;
+try_bin(Str) when is_list(Str) -> list_to_binary(Str);
+try_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
+try_bin(Int) when is_integer(Int) -> integer_to_binary(Int);
+try_bin(Float) when is_float(Float) -> float_to_binary(Float);
+try_bin(Term) -> Term.
+
 bin(Bin) when is_binary(Bin) -> Bin;
 bin(Str) when is_list(Str) -> list_to_binary(Str);
 bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
@@ -699,3 +757,98 @@ atom_conf_path(Path, ExpFun, OnFail) ->
                     error(Err)
             end
     end.
+
+merge_deprecated_cluster_override_to_local_override() ->
+    case read_deprecated_override_conf() of
+        DeprecatedConf when DeprecatedConf =/= #{} ->
+            LocalOverrides = read_override_conf(#{override_to => local}),
+            MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf),
+            _ = save_to_override_conf(MergedConf, #{override_to => local}),
+            ok;
+        _ ->
+            ok
+    end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+remove_default_conf_test() ->
+    ?assertEqual(
+        #{},
+        remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => 100})
+    ),
+    ?assertEqual(
+        #{},
+        remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => <<"100">>})
+    ),
+    ?assertEqual(
+        #{<<"def">> => 100},
+        remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => #{<<"bar">> => 100}})
+    ),
+    ?assertEqual(
+        #{<<"def">> => #{<<"edf">> => 321}},
+        remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{
+            <<"def">> => #{<<"abc">> => 100, <<"edf">> => 123}
+        })
+    ),
+    ?assertEqual(
+        #{},
+        remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{
+            <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => 321}
+        })
+    ),
+    ?assertEqual(
+        #{},
+        remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => <<"true">>}}, #{
+            <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => true}
+        })
+    ),
+    ?assertEqual(
+        #{},
+        remove_default_conf(
+            #{
+                <<"bytes_in">> =>
+                    #{
+                        <<"capacity">> => infinity,
+                        <<"initial">> => 0,
+                        <<"rate">> => infinity
+                    }
+            },
+            #{
+                <<"bytes_in">> =>
+                    #{
+                        <<"capacity">> => <<"infinity">>,
+                        <<"initial">> => <<"0">>,
+                        <<"rate">> => <<"infinity">>
+                    }
+            }
+        )
+    ),
+    ?assertEqual(
+        #{},
+        remove_default_conf(
+            #{
+                <<"limiter">> => #{
+                    <<"connection">> =>
+                        #{
+                            <<"capacity">> => 1000,
+                            <<"initial">> => <<"0">>,
+                            <<"rate">> => <<"1000/s">>
+                        }
+                }
+            },
+            #{
+                <<"limiter">> => #{
+                    <<"connection">> =>
+                        #{
+                            <<"capacity">> => <<"1000">>,
+                            <<"initial">> => <<"0">>,
+                            <<"rate">> => <<"1000/s">>
+                        }
+                }
+            }
+        )
+    ),
+    ok.
+
+-endif.

apps/emqx/src/emqx_config_handler.erl

@@ -271,8 +271,10 @@ do_update_config(
     SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf),
     SubHandlers = get_sub_handlers(ConfKey, Handlers),
     case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of
-        {ok, NewUpdateReq} -> merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
+        {ok, NewUpdateReq} ->
-        Error -> Error
+            merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
+        Error ->
+            Error
     end.
 
 check_and_save_configs(
@@ -289,7 +291,9 @@ check_and_save_configs(
     OldConf = emqx_config:get_root(ConfKeyPath),
     case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
         {ok, Result0} ->
-            ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts),
+            ok = emqx_config:save_configs(
+                ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, Opts
+            ),
             Result1 = return_change_result(ConfKeyPath, UpdateArgs),
             {ok, Result1#{post_config_update => Result0}};
         Error ->
@@ -560,11 +564,11 @@ check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) ->
                             allow;
                         {error, Error} ->
                             ?SLOG(error, #{
-                                msg => "prevent_remove_local_override_conf",
+                                msg => "prevent_remove_local_conf",
                                 config_key_path => ConfKeyPath,
                                 error => Error
                             }),
-                            {deny, "Disable changed from local-override.conf"}
+                            {deny, "Disable changed from local conf"}
                     end;
                 {not_found, _, _} ->
                     allow

apps/emqx/test/emqx_config_handler_SUITE.erl

@@ -21,8 +21,8 @@
 
 -define(MOD, {mod}).
 -define(WKEY, '?').
--define(LOCAL_CONF, "/tmp/local-override.conf").
+-define(LOCAL_CONF, "/tmp/local.conf").
--define(CLUSTER_CONF, "/tmp/cluster-override.conf").
+-define(CLUSTER_CONF, "/tmp/cluster.conf").
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -221,12 +221,6 @@ t_local_override_update_remove(_Config) ->
     ),
     ?assertMatch(0.7, emqx:get_config(KeyPath)),
 
-    KeyPath2 = [sysmon, os, cpu_low_watermark],
-    ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE),
-    ?assertMatch(
-        {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts)
-    ),
-
     %% remove
     ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)),
     ?assertEqual(
@@ -251,7 +245,6 @@ t_local_override_update_remove(_Config) ->
     ?assert(length(OSKey1) > 1),
 
     ok = emqx_config_handler:remove_handler(KeyPath),
-    ok = emqx_config_handler:remove_handler(KeyPath2),
     application:unset_env(emqx, local_override_conf_file),
     application:unset_env(emqx, cluster_override_conf_file),
     ok.
@@ -426,9 +419,9 @@ wait_for_new_pid() ->
 callback_error(FailedPath, Update, Error) ->
     Opts = #{rawconf_with_defaults => true},
     ok = emqx_config_handler:add_handler(FailedPath, ?MODULE),
-    Old = emqx:get_raw_config(FailedPath),
+    Old = emqx:get_raw_config(FailedPath, undefined),
     ?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)),
-    New = emqx:get_raw_config(FailedPath),
+    New = emqx:get_raw_config(FailedPath, undefined),
     ?assertEqual(Old, New),
     ok = emqx_config_handler:remove_handler(FailedPath),
     ok.

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -1076,10 +1076,10 @@ tr_config_files(_Conf) ->
     end.
 
 tr_cluster_override_conf_file(Conf) ->
-    tr_override_conf_file(Conf, "cluster-override.conf").
+    tr_override_conf_file(Conf, "cluster.conf").
 
 tr_local_override_conf_file(Conf) ->
-    tr_override_conf_file(Conf, "local-override.conf").
+    tr_override_conf_file(Conf, "local.conf").
 
 tr_override_conf_file(Conf, Filename) ->
     DataDir = conf_get("node.data_dir", Conf),

apps/emqx_conf/test/emqx_conf_app_SUITE.erl

@@ -85,7 +85,7 @@ create_data_dir() ->
     ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)),
     ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)),
     Telemetry = <<"telemetry.enable = false">>,
-    ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry).
+    ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry).
 
 set_data_dir_env() ->
     Node = atom_to_list(node()),
@@ -100,14 +100,14 @@ set_data_dir_env() ->
     ok = file:write_file(NewConfigFile, DataDir, [append]),
     application:set_env(emqx, config_files, [NewConfigFile]),
     application:set_env(emqx, data_dir, Node),
-    application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"),
+    application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster.hocon"),
     ok.
 
 assert_data_copy_done([First0 | Rest]) ->
     First = atom_to_list(First0),
     {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"),
     {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"),
-    {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster-override.conf"),
+    {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"),
     lists:foreach(
         fun(Node0) ->
             Node = atom_to_list(Node0),
@@ -118,7 +118,7 @@ assert_data_copy_done([First0 | Rest]) ->
             ),
             ?assertEqual(
                 {ok, FakeOverrideFile},
-                file:read_file(Node ++ "/configs/cluster-override.conf"),
+                file:read_file(Node ++ "/configs/cluster.hocon"),
                 #{node => Node}
             ),
             ?assertEqual(

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -31,41 +31,23 @@
     global_zone_configs/3
 ]).
 
--export([gen_schema/1]).
-
 -define(PREFIX, "/configs/").
 -define(PREFIX_RESET, "/configs_reset/").
 -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))).
 -define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}).
 -define(TAGS, ["Configs"]).
 
--define(EXCLUDES,
+-define(ROOT_KEYS, [
-    [
+    <<"dashboard">>,
-        <<"exhook">>,
+    <<"alarm">>,
-        <<"gateway">>,
+    <<"sys_topics">>,
-        <<"plugins">>,
+    <<"sysmon">>,
-        <<"bridges">>,
+    <<"limiter">>,
-        <<"rule_engine">>,
+    <<"trace">>,
-        <<"authorization">>,
+    <<"log">>,
-        <<"authentication">>,
+    <<"persistent_session_store">>,
-        <<"rpc">>,
+    <<"zones">>
-        <<"connectors">>,
+]).
-        <<"slow_subs">>,
-        <<"psk_authentication">>,
-        <<"topic_metrics">>,
-        <<"rewrite">>,
-        <<"auto_subscribe">>,
-        <<"retainer">>,
-        <<"statsd">>,
-        <<"delayed">>,
-        <<"event_message">>,
-        <<"prometheus">>,
-        <<"telemetry">>,
-        <<"listeners">>,
-        <<"license">>,
-        <<"api_key">>
-    ] ++ global_zone_roots()
-).
 
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@@ -119,7 +101,6 @@ schema("/configs_reset/:rootname") ->
                     "- For a config entry that has default value, this resets it to the default value;\n"
                     "- For a config entry that has no default value, an error 400 will be returned"
                 >>,
-            summary => <<"Reset config entry">>,
             %% We only return "200" rather than the new configs that has been changed, as
             %% the schema of the changed configs is depends on the request parameter
             %% `conf_path`, it cannot be defined here.
@@ -214,12 +195,11 @@ fields(Field) ->
 %%%==============================================================================================
 %% HTTP API Callbacks
 config(get, _Params, Req) ->
+    [Path] = conf_path(Req),
+    {200, get_raw_config(Path)};
+config(put, #{body := NewConf}, Req) ->
     Path = conf_path(Req),
-    {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()),
+    case emqx_conf:update(Path, NewConf, ?OPTS) of
-    {200, Conf};
-config(put, #{body := Body}, Req) ->
-    Path = conf_path(Req),
-    case emqx_conf:update(Path, Body, ?OPTS) of
         {ok, #{raw_config := RawConf}} ->
             {200, RawConf};
         {error, {permission_denied, Reason}} ->
@@ -229,28 +209,29 @@ config(put, #{body := Body}, Req) ->
     end.
 
 global_zone_configs(get, _Params, _Req) ->
-    Paths = global_zone_roots(),
+    {200, get_zones()};
-    Zones = lists:foldl(
-        fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end,
-        #{},
-        Paths
-    ),
-    {200, Zones};
 global_zone_configs(put, #{body := Body}, _Req) ->
+    PrevZones = get_zones(),
     Res =
         maps:fold(
             fun(Path, Value, Acc) ->
-                case emqx_conf:update([Path], Value, ?OPTS) of
+                PrevValue = maps:get(Path, PrevZones),
-                    {ok, #{raw_config := RawConf}} ->
+                case Value =/= PrevValue of
-                        Acc#{Path => RawConf};
+                    true ->
-                    {error, Reason} ->
+                        case emqx_conf:update([Path], Value, ?OPTS) of
-                        ?SLOG(error, #{
+                            {ok, #{raw_config := RawConf}} ->
-                            msg => "update global zone failed",
+                                Acc#{Path => RawConf};
-                            reason => Reason,
+                            {error, Reason} ->
-                            path => Path,
+                                ?SLOG(error, #{
-                            value => Value
+                                    msg => "update global zone failed",
-                        }),
+                                    reason => Reason,
-                        Acc
+                                    path => Path,
+                                    value => Value
+                                }),
+                                Acc
+                        end;
+                    false ->
+                        Acc#{Path => Value}
                 end
             end,
             #{},
@@ -298,13 +279,30 @@ conf_path_reset(Req) ->
 
 get_full_config() ->
     emqx_config:fill_defaults(
-        maps:without(
+        maps:with(
-            ?EXCLUDES,
+            ?ROOT_KEYS,
             emqx:get_raw_config([])
         ),
         #{obfuscate_sensitive_values => true}
     ).
 
+get_raw_config(Path) ->
+    #{Path := Conf} =
+        emqx_config:fill_defaults(
+            #{Path => emqx:get_raw_config([Path])},
+            #{obfuscate_sensitive_values => true}
+        ),
+    Conf.
+
+get_zones() ->
+    lists:foldl(
+        fun(Path, Acc) ->
+            maps:merge(Acc, get_config_with_default(Path))
+        end,
+        #{},
+        global_zone_roots()
+    ).
+
 get_config_with_default(Path) ->
     emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}).
 
@@ -317,40 +315,12 @@ conf_path_from_querystr(Req) ->
 config_list() ->
     Mod = emqx_conf:schema_module(),
     Roots = hocon_schema:roots(Mod),
-    lists:foldl(fun(Key, Acc) -> lists:keydelete(Key, 1, Acc) end, Roots, ?EXCLUDES).
+    lists:foldl(fun(Key, Acc) -> [lists:keyfind(Key, 1, Roots) | Acc] end, [], ?ROOT_KEYS).
 
 conf_path(Req) ->
     <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req),
     string:lexemes(Path, "/ ").
 
-%% TODO: generate from hocon schema
-gen_schema(Conf) when is_boolean(Conf) ->
-    with_default_value(#{type => boolean}, Conf);
-gen_schema(Conf) when is_binary(Conf); is_atom(Conf) ->
-    with_default_value(#{type => string}, Conf);
-gen_schema(Conf) when is_number(Conf) ->
-    with_default_value(#{type => number}, Conf);
-gen_schema(Conf) when is_list(Conf) ->
-    case io_lib:printable_unicode_list(Conf) of
-        true ->
-            gen_schema(unicode:characters_to_binary(Conf));
-        false ->
-            #{type => array, items => gen_schema(hd(Conf))}
-    end;
-gen_schema(Conf) when is_map(Conf) ->
-    #{
-        type => object,
-        properties =>
-            maps:map(fun(_K, V) -> gen_schema(V) end, Conf)
-    };
-gen_schema(_Conf) ->
-    %% the conf is not of JSON supported type, it may have been converted
-    %% by the hocon schema
-    #{type => string}.
-
-with_default_value(Type, Value) ->
-    Type#{example => emqx_map_lib:binary_string(Value)}.
-
 global_zone_roots() ->
     lists:map(fun({K, _}) -> K end, global_zone_schema()).
 

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -55,12 +55,17 @@ t_update(_Config) ->
     %% update ok
     {ok, SysMon} = get_config(<<"sysmon">>),
     #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon,
-    NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort),
+    NewSysMon = #{<<"vm">> => #{<<"busy_port">> => not BusyPort}},
     {ok, #{}} = update_config(<<"sysmon">>, NewSysMon),
     {ok, SysMon1} = get_config(<<"sysmon">>),
     #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1,
     ?assertEqual(BusyPort, not BusyPort1),
     assert_busy_port(BusyPort1),
+    %% Make sure the override config is updated, and remove the default value.
+    ?assertEqual(
+        #{<<"vm">> => #{<<"busy_port">> => BusyPort1}},
+        maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster}))
+    ),
 
     %% update failed
     ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"),
@@ -130,6 +135,8 @@ t_global_zone(_Config) ->
     NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
     {ok, #{}} = update_global_zone(NewZones),
     ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])),
+    %% Make sure the override config is updated, and remove the default value.
+    ?assertEqual(#{<<"max_qos_allowed">> => 1}, read_conf(<<"mqtt">>)),
 
     BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3),
     ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)),
@@ -145,6 +152,10 @@ t_global_zone(_Config) ->
     %% the default value is 2
     ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)),
     ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0),
+
+    DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2),
+    {ok, #{}} = update_global_zone(DefaultZones),
+    ?assertEqual(undefined, read_conf(<<"mqtt">>)),
     ok.
 
 get_global_zone() ->
@@ -169,7 +180,7 @@ t_dashboard(_Config) ->
     Https1 = #{enable => true, bind => 18084},
     ?assertMatch(
         {error, {"HTTP/1.1", 400, _}},
-        update_config("dashboard", Dashboard#{<<"https">> => Https1})
+        update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})
     ),
 
     Https2 = #{
@@ -179,35 +190,41 @@ t_dashboard(_Config) ->
         cacertfile => "etc/certs/badcacert.pem",
         certfile => "etc/certs/badcert.pem"
     },
-    Dashboard2 = Dashboard#{listeners => Listeners#{https => Https2}},
+    Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}},
     ?assertMatch(
         {error, {"HTTP/1.1", 400, _}},
         update_config("dashboard", Dashboard2)
     ),
 
-    Keyfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])),
+    KeyFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])),
-    Certfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])),
+    CertFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])),
-    Cacertfile = emqx_common_test_helpers:app_path(
+    CacertFile = emqx_common_test_helpers:app_path(
         emqx, filename:join(["etc", "certs", "cacert.pem"])
     ),
     Https3 = #{
-        enable => true,
+        <<"enable">> => true,
-        bind => 18084,
+        <<"bind">> => 18084,
-        keyfile => Keyfile,
+        <<"keyfile">> => list_to_binary(KeyFile),
-        cacertfile => Cacertfile,
+        <<"cacertfile">> => list_to_binary(CacertFile),
-        certfile => Certfile
+        <<"certfile">> => list_to_binary(CertFile)
     },
-    Dashboard3 = Dashboard#{listeners => Listeners#{https => Https3}},
+    Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}},
     ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)),
 
-    Dashboard4 = Dashboard#{listeners => Listeners#{https => #{enable => false}}},
+    Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"enable">> => false}}},
     ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)),
+    {ok, Dashboard41} = get_config("dashboard"),
+    ?assertEqual(
+        Https3#{<<"enable">> => false},
+        read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]),
+        Dashboard41
+    ),
 
     ?assertMatch({ok, _}, update_config("dashboard", Dashboard)),
 
     {ok, Dashboard1} = get_config("dashboard"),
     ?assertNotEqual(Dashboard, Dashboard1),
-    timer:sleep(1000),
+    timer:sleep(1500),
     ok.
 
 t_configs_node({'init', Config}) ->
@@ -288,3 +305,11 @@ reset_config(Name, Key) ->
         {ok, []} -> ok;
         Error -> Error
     end.
+
+read_conf(RootKeys) when is_list(RootKeys) ->
+    case emqx_config:read_override_conf(#{override_to => cluster}) of
+        undefined -> undefined;
+        Conf -> emqx_map_lib:deep_get(RootKeys, Conf, undefined)
+    end;
+read_conf(RootKey) ->
+    read_conf([RootKey]).