Merge pull request #11036 from zhongwencool/conf-load-cli

feat: support --replace|merge with conf load

apps/emqx_authn/src/emqx_authn.erl

@@ -26,7 +26,7 @@
     get_enabled_authns/0
 ]).
 
--export([merge_config/1, import_config/1]).
+-export([merge_config/1, merge_config_local/2, import_config/1]).
 
 -include("emqx_authn.hrl").
 
@@ -162,3 +162,6 @@ authn_list(Authn) when is_map(Authn) ->
 
 merge_config(AuthNs) ->
     emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}).
+
+merge_config_local(AuthNs, Opts) ->
+    emqx:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}, Opts).

apps/emqx_authz/src/emqx_authz.erl

@@ -33,6 +33,7 @@
     move/2,
     update/2,
     merge/1,
+    merge_local/2,
     authorize/5,
     %% for telemetry information
     get_enabled_authzs/0
@@ -127,6 +128,9 @@ lookup(Type) ->
 merge(NewConf) ->
     emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}).
 
+merge_local(NewConf, Opts) ->
+    emqx:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}, Opts).
+
 move(Type, ?CMD_MOVE_BEFORE(Before)) ->
     emqx_authz_utils:update_config(
         ?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}

apps/emqx_conf/src/emqx_conf_cli.erl

@@ -47,16 +47,20 @@ conf(["show"]) ->
     print_hocon(get_config());
 conf(["show", Key]) ->
     print_hocon(get_config(Key));
-conf(["load", "--auth-chains", AuthChains, Path]) when
+conf(["load", "--replace", Path]) ->
-    AuthChains =:= "replace"; AuthChains =:= "merge"
+    load_config(Path, replace);
-->
+conf(["load", "--merge", Path]) ->
-    load_config(Path, AuthChains);
+    load_config(Path, merge);
 conf(["load", Path]) ->
-    load_config(Path, "replace");
+    load_config(Path, merge);
 conf(["cluster_sync" | Args]) ->
     admins(Args);
+conf(["reload", "--merge"]) ->
+    reload_etc_conf_on_local_node(merge);
+conf(["reload", "--replace"]) ->
+    reload_etc_conf_on_local_node(replace);
 conf(["reload"]) ->
-    reload_etc_conf_on_local_node();
+    conf(["reload", "--merge"]);
 conf(_) ->
     emqx_ctl:usage(usage_conf() ++ usage_sync()).
 
@@ -98,17 +102,21 @@ admins(_) ->
 
 usage_conf() ->
     [
-        {"conf reload", "reload etc/emqx.conf on local node"},
+        {"conf reload --replace|--merge", "reload etc/emqx.conf on local node"},
-        {"conf show_keys", "Print all config keys"},
+        {"", "The new configuration values will be overlaid on the existing values by default."},
+        {"", "use the --replace flag to replace existing values with the new ones instead."},
+        {"----------------------------------", "------------"},
+        {"conf show_keys", "print all the currently used configuration keys."},
         {"conf show [<key>]",
-            "Print in-use configs (including default values) under the given key. "
+            "Print in-use configs (including default values) under the given key."},
-            "Print ALL keys if key is not provided"},
+        {"", "Print ALL keys if key is not provided"},
-        {"conf load <path>",
+        {"conf load --replace|--merge <path>", "Load a HOCON format config file."},
-            "Load a HOCON format config file."
+        {"", "The new configuration values will be overlaid on the existing values by default."},
-            "The config is overlay on top of the existing configs. "
+        {"", "use the --replace flag to replace existing values with the new ones instead."},
-            "The current node will initiate a cluster wide config change "
+        {"", "The current node will initiate a cluster wide config change"},
-            "transaction to sync the changes to other nodes in the cluster. "
+        {"", "transaction to sync the changes to other nodes in the cluster. "},
-            "NOTE: do not make runtime config changes during rolling upgrade."}
+        {"", "NOTE: do not make runtime config changes during rolling upgrade."},
+        {"----------------------------------", "------------"}
     ].
 
 usage_sync() ->
@@ -162,19 +170,7 @@ drop_hidden_roots(Conf) ->
     maps:without(Hidden, Conf).
 
 hidden_roots() ->
-    SchemaModule = emqx_conf:schema_module(),
+    [trace, stats, broker].
-    Roots = hocon_schema:roots(SchemaModule),
-    lists:filtermap(
-        fun({BinName, {_RefName, Schema}}) ->
-            case hocon_schema:field_schema(Schema, importance) =/= ?IMPORTANCE_HIDDEN of
-                true ->
-                    false;
-                false ->
-                    {true, BinName}
-            end
-        end,
-        Roots
-    ).
 
 get_config(Key) ->
     case emqx:get_raw_config([Key], undefined) of
@@ -183,7 +179,7 @@ get_config(Key) ->
     end.
 
 -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
-load_config(Path, AuthChain) ->
+load_config(Path, ReplaceOrMerge) ->
     case hocon:files([Path]) of
         {ok, RawConf} when RawConf =:= #{} ->
             emqx_ctl:warning("load ~ts is empty~n", [Path]),
@@ -192,7 +188,7 @@ load_config(Path, AuthChain) ->
             case check_config(RawConf) of
                 ok ->
                     lists:foreach(
-                        fun({K, V}) -> update_config(K, V, AuthChain) end,
+                        fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end,
                         to_sorted_list(RawConf)
                     );
                 {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
@@ -216,15 +212,34 @@ load_config(Path, AuthChain) ->
             {error, bad_hocon_file}
     end.
 
-update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
+update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
     check_res(Key, emqx_authz:merge(Conf));
-update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
+update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
     check_res(Key, emqx_authn:merge_config(Conf));
-update_config(Key, Value, _) ->
+update_config_cluster(Key, NewConf, merge) ->
+    Merged = merge_conf(Key, NewConf),
+    check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS));
+update_config_cluster(Key, Value, replace) ->
     check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)).
 
-check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]);
+-define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}).
-check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]).
+update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
+    check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS));
+update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
+    check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS));
+update_config_local(Key, NewConf, merge) ->
+    Merged = merge_conf(Key, NewConf),
+    check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS));
+update_config_local(Key, Value, replace) ->
+    check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS)).
+
+check_res(Key, Res) -> check_res(cluster, Key, Res).
+check_res(Mode, Key, {ok, _} = Res) ->
+    emqx_ctl:print("load ~ts in ~p ok~n", [Key, Mode]),
+    Res;
+check_res(_Mode, Key, {error, Reason} = Res) ->
+    emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]),
+    Res.
 
 check_config(Conf) ->
     case check_keys_is_not_readonly(Conf) of
@@ -252,15 +267,19 @@ check_config_schema(Conf) ->
     sorted_fold(Fold, Conf).
 
 %% @doc Reload etc/emqx.conf to runtime config except for the readonly config
--spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
+-spec reload_etc_conf_on_local_node(replace | merge) -> ok | {error, term()}.
-reload_etc_conf_on_local_node() ->
+reload_etc_conf_on_local_node(ReplaceOrMerge) ->
     case load_etc_config_file() of
         {ok, RawConf} ->
-            case check_readonly_config(RawConf) of
+            case filter_readonly_config(RawConf) of
-                {ok, Reloaded} -> reload_config(Reloaded);
+                {ok, Reloaded} ->
-                {error, Error} -> {error, Error}
+                    reload_config(Reloaded, ReplaceOrMerge);
+                {error, Error} ->
+                    emqx_ctl:warning("check config failed~n~p~n", [Error]),
+                    {error, Error}
             end;
-        {error, _Error} ->
+        {error, Error} ->
+            emqx_ctl:warning("bad_hocon_file~n ~p~n", [Error]),
             {error, bad_hocon_file}
     end.
 
@@ -290,23 +309,13 @@ load_etc_config_file() ->
             {error, Error}
     end.
 
-check_readonly_config(Raw) ->
+filter_readonly_config(Raw) ->
     SchemaMod = emqx_conf:schema_module(),
     RawDefault = emqx_config:fill_defaults(Raw),
     case emqx_conf:check_config(SchemaMod, RawDefault) of
-        {ok, CheckedConf} ->
+        {ok, _CheckedConf} ->
-            case filter_changed_readonly_keys(CheckedConf) of
+            ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
-                [] ->
+            {ok, maps:without(ReadOnlyKeys, Raw)};
-                    ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
-                    {ok, maps:without(ReadOnlyKeys, Raw)};
-                Error ->
-                    ?SLOG(error, #{
-                        msg => ?UPDATE_READONLY_KEYS_PROHIBITED,
-                        read_only_keys => ?READONLY_KEYS,
-                        error => Error
-                    }),
-                    {error, Error}
-            end;
         {error, Error} ->
             ?SLOG(error, #{
                 msg => "bad_etc_config_schema_found",
@@ -315,14 +324,12 @@ check_readonly_config(Raw) ->
             {error, Error}
     end.
 
-reload_config(AllConf) ->
+reload_config(AllConf, ReplaceOrMerge) ->
     Fold = fun({Key, Conf}, Acc) ->
-        case emqx:update_config([Key], Conf, #{persistent => false}) of
+        case update_config_local(Key, Conf, ReplaceOrMerge) of
             {ok, _} ->
-                emqx_ctl:print("Reloaded ~ts config ok~n", [Key]),
                 Acc;
             Error ->
-                emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]),
                 ?SLOG(error, #{
                     msg => "failed_to_reload_etc_config",
                     key => Key,
@@ -334,21 +341,6 @@ reload_config(AllConf) ->
     end,
     sorted_fold(Fold, AllConf).
 
-filter_changed_readonly_keys(Conf) ->
-    lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS).
-
-filter_changed(Key, ChangedConf) ->
-    Prev = emqx_conf:get([Key], #{}),
-    New = maps:get(Key, ChangedConf, #{}),
-    case Prev =/= New of
-        true -> {true, {Key, changed(New, Prev)}};
-        false -> false
-    end.
-
-changed(New, Prev) ->
-    Diff = emqx_utils_maps:diff_maps(New, Prev),
-    maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
-
 sorted_fold(Func, Conf) ->
     case lists:foldl(Func, [], to_sorted_list(Conf)) of
         [] -> ok;
@@ -357,3 +349,12 @@ sorted_fold(Func, Conf) ->
 
 to_sorted_list(Conf) ->
     lists:keysort(1, maps:to_list(Conf)).
+
+merge_conf(Key, NewConf) ->
+    OldConf = emqx_conf:get_raw([Key]),
+    do_merge_conf(OldConf, NewConf).
+
+do_merge_conf(OldConf = #{}, NewConf = #{}) ->
+    emqx_utils_maps:deep_merge(OldConf, NewConf);
+do_merge_conf(_OldConf, NewConf) ->
+    NewConf.

apps/emqx_conf/test/emqx_conf_cli_SUITE.erl

@@ -33,24 +33,24 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
 
-t_load_config(Config) ->
+t_load_config_with(Config) ->
     Authz = authorization,
     Conf = emqx_conf:get_raw([Authz]),
     %% set sources to []
-    ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
+    ConfBin0 = hocon_pp:do(#{<<"authorization">> => #{<<"sources">> => []}}, #{}),
     ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
-    ok = emqx_conf_cli:conf(["load", ConfFile0]),
+    ok = emqx_conf_cli:conf(["load", "--merge", ConfFile0]),
     ?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])),
     %% remove sources, it will reset to default file source.
     ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
     ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
-    ok = emqx_conf_cli:conf(["load", ConfFile1]),
+    ok = emqx_conf_cli:conf(["load", "--replace", ConfFile1]),
     Default = [emqx_authz_schema:default_authz()],
     ?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])),
     %% reset
     ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
     ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
-    ok = emqx_conf_cli:conf(["load", ConfFile2]),
+    ok = emqx_conf_cli:conf(["load", "--replace", ConfFile2]),
     ?assertEqual(
         Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]},
         emqx_conf:get_raw([Authz])
@@ -75,7 +75,9 @@ t_load_readonly(Config) ->
             ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}),
             ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
             application:set_env(emqx, config_files, [ConfFile1]),
-            ?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"]))
+            ?assertMatch(ok, emqx_conf_cli:conf(["reload"])),
+            %% Don't update readonly key
+            ?assertEqual(Conf, emqx_conf:get_raw([Key]))
         end,
         ?READONLY_KEYS
     ),