feat: add conf reload cli

apps/emqx/include/emqx.hrl

@@ -17,6 +17,9 @@
 -ifndef(EMQX_HRL).
 -define(EMQX_HRL, true).
 
+%% Config
+-define(READ_ONLY_KEYS, [cluster, rpc, node]).
+
 %% Shard
 %%--------------------------------------------------------------------
 -define(COMMON_SHARD, emqx_common_shard).

apps/emqx/src/emqx_config.erl

@@ -18,6 +18,7 @@
 -compile({no_auto_import, [get/0, get/1, put/2, erase/1]}).
 -elvis([{elvis_style, god_modules, disable}]).
 -include("logger.hrl").
+-include("emqx.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -export([
@@ -33,7 +34,8 @@
     save_configs/5,
     save_to_app_env/1,
     save_to_config_map/2,
-    save_to_override_conf/3
+    save_to_override_conf/3,
+    reload_etc_conf_on_local_node/0
 ]).
 -export([merge_envs/2]).
 
@@ -311,8 +313,7 @@ put_raw(KeyPath0, Config) ->
 %% Load/Update configs From/To files
 %%============================================================================
 init_load(SchemaMod) ->
-    ConfFiles = application:get_env(emqx, config_files, []),
+    init_load(SchemaMod, config_files()).
-    init_load(SchemaMod, ConfFiles).
 
 %% @doc Initial load of the given config files.
 %% NOTE: The order of the files is significant, configs from files ordered
@@ -975,3 +976,95 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
     ok;
 put_config_post_change_actions(_Key, _NewValue) ->
     ok.
+
+%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
+-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
+reload_etc_conf_on_local_node() ->
+    case load_etc_config_file() of
+        {ok, RawConf} ->
+            case check_readonly_config(RawConf) of
+                {ok, Reloaded} -> reload_config(Reloaded);
+                {error, Error} -> {error, Error}
+            end;
+        {error, _Error} ->
+            {error, bad_hocon_file}
+    end.
+
+reload_config(AllConf) ->
+    Func = fun(Key, Conf, Acc) ->
+        case emqx:update_config([Key], Conf, #{persistent => false}) of
+            {ok, _} ->
+                io:format("Reloaded ~ts config ok~n", [Key]),
+                Acc;
+            Error ->
+                ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
+                ?SLOG(error, #{
+                    msg => "failed_to_reload_etc_config",
+                    key => Key,
+                    value => Conf,
+                    error => Error
+                }),
+                Acc#{Key => Error}
+        end
+    end,
+    Res = maps:fold(Func, #{}, AllConf),
+    case Res =:= #{} of
+        true -> ok;
+        false -> {error, Res}
+    end.
+
+%% @doc Merge etc/emqx.conf on top of cluster.hocon.
+%% For example:
+%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
+%% but `authorization.sources` in not in the default emqx.conf file.
+%% To make sure all root keys in emqx.conf has a fully merged value.
+load_etc_config_file() ->
+    ConfFiles = config_files(),
+    Opts = #{format => map, include_dirs => include_dirs()},
+    case hocon:files(ConfFiles, Opts) of
+        {ok, RawConf} ->
+            HasDeprecatedFile = has_deprecated_file(),
+            %% Merge etc.conf on top of cluster.hocon,
+            %% Don't use map deep_merge, use hocon files merge instead.
+            %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
+            Keys = maps:keys(RawConf),
+            MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles),
+            {ok, maps:with(Keys, MergedRaw)};
+        {error, Error} ->
+            ?SLOG(error, #{
+                msg => "failed_to_read_etc_config",
+                files => ConfFiles,
+                error => Error
+            }),
+            {error, Error}
+    end.
+
+check_readonly_config(Raw) ->
+    SchemaMod = emqx_conf:schema_module(),
+    {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}),
+    case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of
+        [] ->
+            {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)};
+        Error ->
+            ?SLOG(error, #{
+                msg => "failed_to_change_read_only_key_in_etc_config",
+                read_only_keys => ?READ_ONLY_KEYS,
+                error => Error
+            }),
+            {error, Error}
+    end.
+
+filter_changed(Key, ChangedConf) ->
+    Prev = get([Key], #{}),
+    New = maps:get(Key, ChangedConf, #{}),
+    case Prev =/= New of
+        true -> {true, {Key, changed(New, Prev)}};
+        false -> false
+    end.
+
+changed(New, Prev) ->
+    Diff = emqx_utils_maps:diff_maps(New, Prev),
+    maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
+
+config_files() ->
+    application:get_env(emqx, config_files, []).

apps/emqx_authz/src/emqx_authz.erl

@@ -166,10 +166,10 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
 do_pre_config_replace(Conf, Conf) ->
     Conf;
 do_pre_config_replace(NewConf, OldConf) ->
-    #{<<"sources">> := NewSources} = NewConf,
+    NewSources = maps:get(<<"sources">>, NewConf, []),
-    #{<<"sources">> := OldSources} = OldConf,
+    OldSources = maps:get(<<"sources">>, OldConf, []),
     NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
-    NewConf#{<<"sources">> := NewSources1}.
+    NewConf#{<<"sources">> => NewSources1}.
 
 do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) ->
     do_move(Cmd, Sources);

apps/emqx_conf/src/emqx_conf_cli.erl

@@ -15,6 +15,7 @@
 %%--------------------------------------------------------------------
 
 -module(emqx_conf_cli).
+-include_lib("emqx/include/emqx.hrl").
 -export([
     load/0,
     admins/1,
@@ -87,8 +88,7 @@ admins(_) ->
 
 usage_conf() ->
     [
-        %% TODO add reload
+        {"conf reload", "reload etc/emqx.conf on local node"},
-        %{"conf reload", "reload etc/emqx.conf on local node"},
         {"conf show_keys", "Print all config keys"},
         {"conf show [<key>]",
             "Print in-use configs (including default values) under the given key. "
@@ -138,11 +138,14 @@ print_keys(Config) ->
 print(Json) ->
     emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]).
 
-print_hocon(Hocon) ->
+print_hocon(Hocon) when is_map(Hocon) ->
-    emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]).
+    emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]);
+print_hocon({error, Error}) ->
+    emqx_ctl:warning("~ts~n", [Error]).
 
 get_config() ->
-    drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))).
+    AllConf = emqx_config:fill_defaults(emqx:get_raw_config([])),
+    drop_hidden_roots(AllConf).
 
 drop_hidden_roots(Conf) ->
     Hidden = hidden_roots(),
@@ -164,22 +167,41 @@ hidden_roots() ->
     ).
 
 get_config(Key) ->
-    emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}).
+    case emqx:get_raw_config(Key, undefined) of
+        undefined -> {error, "key_not_found"};
+        Value -> emqx_config:fill_defaults(#{Key => Value})
+    end.
 
 -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
 load_config(Path) ->
     case hocon:files([Path]) of
-        {ok, Conf} ->
+        {ok, RawConf} ->
-            maps:foreach(
+            case check_config_keys(RawConf) of
-                fun(Key, Value) ->
+                ok ->
-                    case emqx_conf:update([Key], Value, ?OPTIONS) of
+                    maps:foreach(fun update_config/2, RawConf);
-                        {ok, _} -> emqx_ctl:print("load ~ts ok~n", [Key]);
-                        {error, Reason} -> emqx_ctl:print("load ~ts failed: ~p~n", [Key, Reason])
-                    end
-                end,
-                Conf
-            );
                 {error, Reason} ->
-            emqx_ctl:print("load ~ts failed~n~p~n", [Path, Reason]),
+                    emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]),
+                    emqx_ctl:warning(
+                        "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n"
+                    ),
+                    {error, Reason}
+            end;
+        {error, Reason} ->
+            emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]),
             {error, bad_hocon_file}
     end.
+
+update_config(Key, Value) ->
+    case emqx_conf:update([Key], Value, ?OPTIONS) of
+        {ok, _} ->
+            emqx_ctl:print("load ~ts in cluster ok~n", [Key]);
+        {error, Reason} ->
+            emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason])
+    end.
+check_config_keys(Conf) ->
+    Keys = maps:keys(Conf),
+    ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS],
+    case ReadOnlyKeys -- Keys of
+        ReadOnlyKeys -> ok;
+        _ -> {error, "update_read_only_keys_prohibited"}
+    end.

apps/emqx_ctl/src/emqx_ctl.erl

@@ -38,6 +38,8 @@
 -export([
     print/1,
     print/2,
+    warning/1,
+    warning/2,
     usage/1,
     usage/2
 ]).
@@ -180,6 +182,14 @@ print(Msg) ->
 print(Format, Args) ->
     io:format("~ts", [format(Format, Args)]).
 
+-spec warning(io:format()) -> ok.
+warning(Format) ->
+    warning(Format, []).
+
+-spec warning(io:format(), [term()]) -> ok.
+warning(Format, Args) ->
+    io:format("\e[31m~ts\e[0m", [format(Format, Args)]).
+
 -spec usage([cmd_usage()]) -> ok.
 usage(UsageList) ->
     io:format(format_usage(UsageList)).