yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11036 from zhongwencool/conf-load-cli 

feat: support --replace|merge with conf load
This commit is contained in:
zhongwencool 2023-06-14 10:48:13 +08:00 committed by GitHub
parent 7f0f40cb58 9b5c7065f2
commit d41a23b7d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 92 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/emqx_authn/src/emqx_authn.erl
View File

@ -26,7 +26,7 @@
get_enabled_authns/0
]).
-export([merge_config/1, import_config/1]).
-export([merge_config/1, merge_config_local/2, import_config/1]).
-include("emqx_authn.hrl").
@ -162,3 +162,6 @@ authn_list(Authn) when is_map(Authn) ->
merge_config(AuthNs) ->
emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}).
merge_config_local(AuthNs, Opts) ->
emqx:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}, Opts).

4
apps/emqx_authz/src/emqx_authz.erl
View File

@ -33,6 +33,7 @@
move/2,
update/2,
merge/1,
merge_local/2,
authorize/5,
%% for telemetry information
get_enabled_authzs/0
@ -127,6 +128,9 @@ lookup(Type) ->
merge(NewConf) ->
emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}).
merge_local(NewConf, Opts) ->
emqx:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}, Opts).
move(Type, ?CMD_MOVE_BEFORE(Before)) ->
emqx_authz_utils:update_config(
?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}

151
apps/emqx_conf/src/emqx_conf_cli.erl
View File

@ -47,16 +47,20 @@ conf(["show"]) ->
print_hocon(get_config());
conf(["show", Key]) ->
print_hocon(get_config(Key));
conf(["load", "--auth-chains", AuthChains, Path]) when
AuthChains =:= "replace"; AuthChains =:= "merge"
->
load_config(Path, AuthChains);
conf(["load", "--replace", Path]) ->
load_config(Path, replace);
conf(["load", "--merge", Path]) ->
load_config(Path, merge);
conf(["load", Path]) ->
load_config(Path, "replace");
load_config(Path, merge);
conf(["cluster_sync" | Args]) ->
admins(Args);
conf(["reload", "--merge"]) ->
reload_etc_conf_on_local_node(merge);
conf(["reload", "--replace"]) ->
reload_etc_conf_on_local_node(replace);
conf(["reload"]) ->
reload_etc_conf_on_local_node();
conf(["reload", "--merge"]);
conf(_) ->
emqx_ctl:usage(usage_conf() ++ usage_sync()).
@ -98,17 +102,21 @@ admins(_) ->
usage_conf() ->
[
{"conf reload", "reload etc/emqx.conf on local node"},
{"conf show_keys", "Print all config keys"},
{"conf reload --replace|--merge", "reload etc/emqx.conf on local node"},
{"", "The new configuration values will be overlaid on the existing values by default."},
{"", "use the --replace flag to replace existing values with the new ones instead."},
{"----------------------------------", "------------"},
{"conf show_keys", "print all the currently used configuration keys."},
{"conf show [<key>]",
"Print in-use configs (including default values) under the given key. "
"Print ALL keys if key is not provided"},
{"conf load <path>",
"Load a HOCON format config file."
"The config is overlay on top of the existing configs. "
"The current node will initiate a cluster wide config change "
"transaction to sync the changes to other nodes in the cluster. "
"NOTE: do not make runtime config changes during rolling upgrade."}
"Print in-use configs (including default values) under the given key."},
{"", "Print ALL keys if key is not provided"},
{"conf load --replace|--merge <path>", "Load a HOCON format config file."},
{"", "The new configuration values will be overlaid on the existing values by default."},
{"", "use the --replace flag to replace existing values with the new ones instead."},
{"", "The current node will initiate a cluster wide config change"},
{"", "transaction to sync the changes to other nodes in the cluster. "},
{"", "NOTE: do not make runtime config changes during rolling upgrade."},
{"----------------------------------", "------------"}
].
usage_sync() ->
@ -162,19 +170,7 @@ drop_hidden_roots(Conf) ->
maps:without(Hidden, Conf).
hidden_roots() ->
SchemaModule = emqx_conf:schema_module(),
Roots = hocon_schema:roots(SchemaModule),
lists:filtermap(
fun({BinName, {_RefName, Schema}}) ->
case hocon_schema:field_schema(Schema, importance) =/= ?IMPORTANCE_HIDDEN of
true ->
false;
false ->
{true, BinName}
end
end,
Roots
).
[trace, stats, broker].
get_config(Key) ->
case emqx:get_raw_config([Key], undefined) of
@ -183,7 +179,7 @@ get_config(Key) ->
end.
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
load_config(Path, AuthChain) ->
load_config(Path, ReplaceOrMerge) ->
case hocon:files([Path]) of
{ok, RawConf} when RawConf =:= #{} ->
emqx_ctl:warning("load ~ts is empty~n", [Path]),
@ -192,7 +188,7 @@ load_config(Path, AuthChain) ->
case check_config(RawConf) of
ok ->
lists:foreach(
fun({K, V}) -> update_config(K, V, AuthChain) end,
fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end,
to_sorted_list(RawConf)
);
{error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
@ -216,15 +212,34 @@ load_config(Path, AuthChain) ->
{error, bad_hocon_file}
end.
update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
check_res(Key, emqx_authz:merge(Conf));
update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
check_res(Key, emqx_authn:merge_config(Conf));
update_config(Key, Value, _) ->
update_config_cluster(Key, NewConf, merge) ->
Merged = merge_conf(Key, NewConf),
check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS));
update_config_cluster(Key, Value, replace) ->
check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)).
check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]);
check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]).
-define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}).
update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS));
update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) ->
check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS));
update_config_local(Key, NewConf, merge) ->
Merged = merge_conf(Key, NewConf),
check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS));
update_config_local(Key, Value, replace) ->
check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS)).
check_res(Key, Res) -> check_res(cluster, Key, Res).
check_res(Mode, Key, {ok, _} = Res) ->
emqx_ctl:print("load ~ts in ~p ok~n", [Key, Mode]),
Res;
check_res(_Mode, Key, {error, Reason} = Res) ->
emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]),
Res.
check_config(Conf) ->
case check_keys_is_not_readonly(Conf) of
@ -252,15 +267,19 @@ check_config_schema(Conf) ->
sorted_fold(Fold, Conf).
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
reload_etc_conf_on_local_node() ->
-spec reload_etc_conf_on_local_node(replace | merge) -> ok | {error, term()}.
reload_etc_conf_on_local_node(ReplaceOrMerge) ->
case load_etc_config_file() of
{ok, RawConf} ->
case check_readonly_config(RawConf) of
{ok, Reloaded} -> reload_config(Reloaded);
{error, Error} -> {error, Error}
case filter_readonly_config(RawConf) of
{ok, Reloaded} ->
reload_config(Reloaded, ReplaceOrMerge);
{error, Error} ->
emqx_ctl:warning("check config failed~n~p~n", [Error]),
{error, Error}
end;
{error, _Error} ->
{error, Error} ->
emqx_ctl:warning("bad_hocon_file~n ~p~n", [Error]),
{error, bad_hocon_file}
end.
@ -290,23 +309,13 @@ load_etc_config_file() ->
{error, Error}
end.
check_readonly_config(Raw) ->
filter_readonly_config(Raw) ->
SchemaMod = emqx_conf:schema_module(),
RawDefault = emqx_config:fill_defaults(Raw),
case emqx_conf:check_config(SchemaMod, RawDefault) of
{ok, CheckedConf} ->
case filter_changed_readonly_keys(CheckedConf) of
[] ->
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
{ok, maps:without(ReadOnlyKeys, Raw)};
Error ->
?SLOG(error, #{
msg => ?UPDATE_READONLY_KEYS_PROHIBITED,
read_only_keys => ?READONLY_KEYS,
error => Error
}),
{error, Error}
end;
{ok, _CheckedConf} ->
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
{ok, maps:without(ReadOnlyKeys, Raw)};
{error, Error} ->
?SLOG(error, #{
msg => "bad_etc_config_schema_found",
@ -315,14 +324,12 @@ check_readonly_config(Raw) ->
{error, Error}
end.
reload_config(AllConf) ->
reload_config(AllConf, ReplaceOrMerge) ->
Fold = fun({Key, Conf}, Acc) ->
case emqx:update_config([Key], Conf, #{persistent => false}) of
case update_config_local(Key, Conf, ReplaceOrMerge) of
{ok, _} ->
emqx_ctl:print("Reloaded ~ts config ok~n", [Key]),
Acc;
Error ->
emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]),
?SLOG(error, #{
msg => "failed_to_reload_etc_config",
key => Key,
@ -334,21 +341,6 @@ reload_config(AllConf) ->
end,
sorted_fold(Fold, AllConf).
filter_changed_readonly_keys(Conf) ->
lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS).
filter_changed(Key, ChangedConf) ->
Prev = emqx_conf:get([Key], #{}),
New = maps:get(Key, ChangedConf, #{}),
case Prev =/= New of
true -> {true, {Key, changed(New, Prev)}};
false -> false
end.
changed(New, Prev) ->
Diff = emqx_utils_maps:diff_maps(New, Prev),
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
sorted_fold(Func, Conf) ->
case lists:foldl(Func, [], to_sorted_list(Conf)) of
[] -> ok;
@ -357,3 +349,12 @@ sorted_fold(Func, Conf) ->
to_sorted_list(Conf) ->
lists:keysort(1, maps:to_list(Conf)).
merge_conf(Key, NewConf) ->
OldConf = emqx_conf:get_raw([Key]),
do_merge_conf(OldConf, NewConf).
do_merge_conf(OldConf = #{}, NewConf = #{}) ->
emqx_utils_maps:deep_merge(OldConf, NewConf);
do_merge_conf(_OldConf, NewConf) ->
NewConf.

14
apps/emqx_conf/test/emqx_conf_cli_SUITE.erl
View File

@ -33,24 +33,24 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
t_load_config(Config) ->
t_load_config_with(Config) ->
Authz = authorization,
Conf = emqx_conf:get_raw([Authz]),
%% set sources to []
ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
ConfBin0 = hocon_pp:do(#{<<"authorization">> => #{<<"sources">> => []}}, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
ok = emqx_conf_cli:conf(["load", ConfFile0]),
ok = emqx_conf_cli:conf(["load", "--merge", ConfFile0]),
?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])),
%% remove sources, it will reset to default file source.
ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
ok = emqx_conf_cli:conf(["load", ConfFile1]),
ok = emqx_conf_cli:conf(["load", "--replace", ConfFile1]),
Default = [emqx_authz_schema:default_authz()],
?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])),
%% reset
ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
ok = emqx_conf_cli:conf(["load", ConfFile2]),
ok = emqx_conf_cli:conf(["load", "--replace", ConfFile2]),
?assertEqual(
Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]},
emqx_conf:get_raw([Authz])
@ -75,7 +75,9 @@ t_load_readonly(Config) ->
ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}),
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
application:set_env(emqx, config_files, [ConfFile1]),
?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"]))
?assertMatch(ok, emqx_conf_cli:conf(["reload"])),
%% Don't update readonly key
?assertEqual(Conf, emqx_conf:get_raw([Key]))
end,
?READONLY_KEYS
),