diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 64cd4687b..75a4a7714 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -17,6 +17,9 @@ -ifndef(EMQX_HRL). -define(EMQX_HRL, true). +%% Config +-define(READ_ONLY_KEYS, [cluster, rpc, node]). + %% Shard %%-------------------------------------------------------------------- -define(COMMON_SHARD, emqx_common_shard). diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index ff241ed0e..790f6040d 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -18,6 +18,7 @@ -compile({no_auto_import, [get/0, get/1, put/2, erase/1]}). -elvis([{elvis_style, god_modules, disable}]). -include("logger.hrl"). +-include("emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ @@ -33,7 +34,8 @@ save_configs/5, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/3 + save_to_override_conf/3, + reload_etc_conf_on_local_node/0 ]). -export([merge_envs/2]). @@ -311,8 +313,7 @@ put_raw(KeyPath0, Config) -> %% Load/Update configs From/To files %%============================================================================ init_load(SchemaMod) -> - ConfFiles = application:get_env(emqx, config_files, []), - init_load(SchemaMod, ConfFiles). + init_load(SchemaMod, config_files()). %% @doc Initial load of the given config files. %% NOTE: The order of the files is significant, configs from files ordered @@ -975,3 +976,95 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) -> ok; put_config_post_change_actions(_Key, _NewValue) -> ok. + +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} + end. + +reload_config(AllConf) -> + Func = fun(Key, Conf, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + io:format("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + Acc#{Key => Error} + end + end, + Res = maps:fold(Func, #{}, AllConf), + case Res =:= #{} of + true -> ok; + false -> {error, Res} + end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = config_files(), + Opts = #{format => map, include_dirs => include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = emqx_conf:schema_module(), + {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}), + case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of + [] -> + {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)}; + Error -> + ?SLOG(error, #{ + msg => "failed_to_change_read_only_key_in_etc_config", + read_only_keys => ?READ_ONLY_KEYS, + error => Error + }), + {error, Error} + end. + +filter_changed(Key, ChangedConf) -> + Prev = get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + +config_files() -> + application:get_env(emqx, config_files, []). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 3c9698de0..9dd46c0c2 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -166,10 +166,10 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - #{<<"sources">> := NewSources} = NewConf, - #{<<"sources">> := OldSources} = OldConf, + NewSources = maps:get(<<"sources">>, NewConf, []), + OldSources = maps:get(<<"sources">>, OldConf, []), NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), - NewConf#{<<"sources">> := NewSources1}. + NewConf#{<<"sources">> => NewSources1}. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 530e4bfcb..8d74c4cd9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_conf_cli). +-include_lib("emqx/include/emqx.hrl"). -export([ load/0, admins/1, @@ -87,8 +88,7 @@ admins(_) -> usage_conf() -> [ - %% TODO add reload - %{"conf reload", "reload etc/emqx.conf on local node"}, + {"conf reload", "reload etc/emqx.conf on local node"}, {"conf show_keys", "Print all config keys"}, {"conf show []", "Print in-use configs (including default values) under the given key. " @@ -138,11 +138,14 @@ print_keys(Config) -> print(Json) -> emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]). -print_hocon(Hocon) -> - emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]). +print_hocon(Hocon) when is_map(Hocon) -> + emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]); +print_hocon({error, Error}) -> + emqx_ctl:warning("~ts~n", [Error]). get_config() -> - drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))). + AllConf = emqx_config:fill_defaults(emqx:get_raw_config([])), + drop_hidden_roots(AllConf). drop_hidden_roots(Conf) -> Hidden = hidden_roots(), @@ -164,22 +167,41 @@ hidden_roots() -> ). get_config(Key) -> - emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}). + case emqx:get_raw_config(Key, undefined) of + undefined -> {error, "key_not_found"}; + Value -> emqx_config:fill_defaults(#{Key => Value}) + end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). load_config(Path) -> case hocon:files([Path]) of - {ok, Conf} -> - maps:foreach( - fun(Key, Value) -> - case emqx_conf:update([Key], Value, ?OPTIONS) of - {ok, _} -> emqx_ctl:print("load ~ts ok~n", [Key]); - {error, Reason} -> emqx_ctl:print("load ~ts failed: ~p~n", [Key, Reason]) - end - end, - Conf - ); + {ok, RawConf} -> + case check_config_keys(RawConf) of + ok -> + maps:foreach(fun update_config/2, RawConf); + {error, Reason} -> + emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), + emqx_ctl:warning( + "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" + ), + {error, Reason} + end; {error, Reason} -> - emqx_ctl:print("load ~ts failed~n~p~n", [Path, Reason]), + emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), {error, bad_hocon_file} end. + +update_config(Key, Value) -> + case emqx_conf:update([Key], Value, ?OPTIONS) of + {ok, _} -> + emqx_ctl:print("load ~ts in cluster ok~n", [Key]); + {error, Reason} -> + emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]) + end. +check_config_keys(Conf) -> + Keys = maps:keys(Conf), + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS], + case ReadOnlyKeys -- Keys of + ReadOnlyKeys -> ok; + _ -> {error, "update_read_only_keys_prohibited"} + end. diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 76068d361..d1a7ed1d7 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -38,6 +38,8 @@ -export([ print/1, print/2, + warning/1, + warning/2, usage/1, usage/2 ]). @@ -180,6 +182,14 @@ print(Msg) -> print(Format, Args) -> io:format("~ts", [format(Format, Args)]). +-spec warning(io:format()) -> ok. +warning(Format) -> + warning(Format, []). + +-spec warning(io:format(), [term()]) -> ok. +warning(Format, Args) -> + io:format("\e[31m~ts\e[0m", [format(Format, Args)]). + -spec usage([cmd_usage()]) -> ok. usage(UsageList) -> io:format(format_usage(UsageList)).