From a8959900d0c6affd55a93cd5eadde202c77558ea Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 13 Jun 2023 18:00:54 +0800 Subject: [PATCH 1/3] feat: support --replace|merge with conf load --- apps/emqx_conf/src/emqx_conf_cli.erl | 69 ++++++++++++--------- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 14 +++-- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index f2aeec7fb..3f9a9f1ae 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -47,12 +47,12 @@ conf(["show"]) -> print_hocon(get_config()); conf(["show", Key]) -> print_hocon(get_config(Key)); -conf(["load", "--auth-chains", AuthChains, Path]) when - AuthChains =:= "replace"; AuthChains =:= "merge" --> - load_config(Path, AuthChains); +conf(["load", "--replace", Path]) -> + load_config(Path, replace); +conf(["load", "--merge", Path]) -> + load_config(Path, merge); conf(["load", Path]) -> - load_config(Path, "replace"); + load_config(Path, merge); conf(["cluster_sync" | Args]) -> admins(Args); conf(["reload"]) -> @@ -99,13 +99,14 @@ admins(_) -> usage_conf() -> [ {"conf reload", "reload etc/emqx.conf on local node"}, - {"conf show_keys", "Print all config keys"}, + {"conf show_keys", "print all the currently used configuration keys."}, {"conf show []", "Print in-use configs (including default values) under the given key. " "Print ALL keys if key is not provided"}, - {"conf load ", + {"conf load --replace|--merge ", "Load a HOCON format config file." - "The config is overlay on top of the existing configs. " + "The new configuration values will be overlaid on the existing values by default." + "use the --replace flag to replace existing values with the new ones instead." "The current node will initiate a cluster wide config change " "transaction to sync the changes to other nodes in the cluster. " "NOTE: do not make runtime config changes during rolling upgrade."} @@ -183,7 +184,7 @@ get_config(Key) -> end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -load_config(Path, AuthChain) -> +load_config(Path, ReplaceOrMerge) -> case hocon:files([Path]) of {ok, RawConf} when RawConf =:= #{} -> emqx_ctl:warning("load ~ts is empty~n", [Path]), @@ -192,7 +193,7 @@ load_config(Path, AuthChain) -> case check_config(RawConf) of ok -> lists:foreach( - fun({K, V}) -> update_config(K, V, AuthChain) end, + fun({K, V}) -> update_config(K, V, ReplaceOrMerge) end, to_sorted_list(RawConf) ); {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> @@ -216,11 +217,14 @@ load_config(Path, AuthChain) -> {error, bad_hocon_file} end. -update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> +update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> check_res(Key, emqx_authz:merge(Conf)); -update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> +update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> check_res(Key, emqx_authn:merge_config(Conf)); -update_config(Key, Value, _) -> +update_config(Key, NewConf, merge) -> + Merged = merge_conf(Key, NewConf), + check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS)); +update_config(Key, Value, replace) -> check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)). check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); @@ -256,11 +260,15 @@ check_config_schema(Conf) -> reload_etc_conf_on_local_node() -> case load_etc_config_file() of {ok, RawConf} -> - case check_readonly_config(RawConf) of - {ok, Reloaded} -> reload_config(Reloaded); - {error, Error} -> {error, Error} + case filter_readonly_config(RawConf) of + {ok, Reloaded} -> + reload_config(Reloaded); + {error, Error} -> + emqx_ctl:warning("check config failed~n~p~n", [Error]), + {error, Error} end; - {error, _Error} -> + {error, Error} -> + emqx_ctl:warning("bad_hocon_file~n ~p~n", [Error]), {error, bad_hocon_file} end. @@ -290,23 +298,13 @@ load_etc_config_file() -> {error, Error} end. -check_readonly_config(Raw) -> +filter_readonly_config(Raw) -> SchemaMod = emqx_conf:schema_module(), RawDefault = emqx_config:fill_defaults(Raw), case emqx_conf:check_config(SchemaMod, RawDefault) of - {ok, CheckedConf} -> - case filter_changed_readonly_keys(CheckedConf) of - [] -> - ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], - {ok, maps:without(ReadOnlyKeys, Raw)}; - Error -> - ?SLOG(error, #{ - msg => ?UPDATE_READONLY_KEYS_PROHIBITED, - read_only_keys => ?READONLY_KEYS, - error => Error - }), - {error, Error} - end; + {ok, _CheckedConf} -> + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], + {ok, maps:without(ReadOnlyKeys, Raw)}; {error, Error} -> ?SLOG(error, #{ msg => "bad_etc_config_schema_found", @@ -357,3 +355,12 @@ sorted_fold(Func, Conf) -> to_sorted_list(Conf) -> lists:keysort(1, maps:to_list(Conf)). + +merge_conf(Key, NewConf) -> + OldConf = emqx_conf:get_raw([Key]), + do_merge_conf(OldConf, NewConf). + +do_merge_conf(OldConf = #{}, NewConf = #{}) -> + emqx_utils_maps:deep_merge(OldConf, NewConf); +do_merge_conf(_OldConf, NewConf) -> + NewConf. diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl index 0e0cc0127..6eca6b620 100644 --- a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -33,24 +33,24 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]). -t_load_config(Config) -> +t_load_config_with(Config) -> Authz = authorization, Conf = emqx_conf:get_raw([Authz]), %% set sources to [] - ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}), + ConfBin0 = hocon_pp:do(#{<<"authorization">> => #{<<"sources">> => []}}, #{}), ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), - ok = emqx_conf_cli:conf(["load", ConfFile0]), + ok = emqx_conf_cli:conf(["load", "--merge", ConfFile0]), ?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])), %% remove sources, it will reset to default file source. ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}), ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), - ok = emqx_conf_cli:conf(["load", ConfFile1]), + ok = emqx_conf_cli:conf(["load", "--replace", ConfFile1]), Default = [emqx_authz_schema:default_authz()], ?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])), %% reset ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}), ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config), - ok = emqx_conf_cli:conf(["load", ConfFile2]), + ok = emqx_conf_cli:conf(["load", "--replace", ConfFile2]), ?assertEqual( Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]}, emqx_conf:get_raw([Authz]) @@ -75,7 +75,9 @@ t_load_readonly(Config) -> ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}), ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), application:set_env(emqx, config_files, [ConfFile1]), - ?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"])) + ?assertMatch(ok, emqx_conf_cli:conf(["reload"])), + %% Don't update readonly key + ?assertEqual(Conf, emqx_conf:get_raw([Key])) end, ?READONLY_KEYS ), From 095fb040c0fe5bf49dbad8a4290aa360b9b10b22 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 13 Jun 2023 18:50:29 +0800 Subject: [PATCH 2/3] chore: format conf cli usage text --- apps/emqx_conf/src/emqx_conf_cli.erl | 47 ++++++---------------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 3f9a9f1ae..6035c1ec6 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -101,15 +101,15 @@ usage_conf() -> {"conf reload", "reload etc/emqx.conf on local node"}, {"conf show_keys", "print all the currently used configuration keys."}, {"conf show []", - "Print in-use configs (including default values) under the given key. " - "Print ALL keys if key is not provided"}, - {"conf load --replace|--merge ", - "Load a HOCON format config file." - "The new configuration values will be overlaid on the existing values by default." - "use the --replace flag to replace existing values with the new ones instead." - "The current node will initiate a cluster wide config change " - "transaction to sync the changes to other nodes in the cluster. " - "NOTE: do not make runtime config changes during rolling upgrade."} + "Print in-use configs (including default values) under the given key."}, + {"", "Print ALL keys if key is not provided"}, + {"conf load --replace|--merge ", "Load a HOCON format config file."}, + {"", "The new configuration values will be overlaid on the existing values by default."}, + {"", "use the --replace flag to replace existing values with the new ones instead."}, + {"", "The current node will initiate a cluster wide config change"}, + {"", "transaction to sync the changes to other nodes in the cluster. "}, + {"", "NOTE: do not make runtime config changes during rolling upgrade."}, + {"----------------------------------", "------------"} ]. usage_sync() -> @@ -163,19 +163,7 @@ drop_hidden_roots(Conf) -> maps:without(Hidden, Conf). hidden_roots() -> - SchemaModule = emqx_conf:schema_module(), - Roots = hocon_schema:roots(SchemaModule), - lists:filtermap( - fun({BinName, {_RefName, Schema}}) -> - case hocon_schema:field_schema(Schema, importance) =/= ?IMPORTANCE_HIDDEN of - true -> - false; - false -> - {true, BinName} - end - end, - Roots - ). + [trace, stats, broker]. get_config(Key) -> case emqx:get_raw_config([Key], undefined) of @@ -332,21 +320,6 @@ reload_config(AllConf) -> end, sorted_fold(Fold, AllConf). -filter_changed_readonly_keys(Conf) -> - lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS). - -filter_changed(Key, ChangedConf) -> - Prev = emqx_conf:get([Key], #{}), - New = maps:get(Key, ChangedConf, #{}), - case Prev =/= New of - true -> {true, {Key, changed(New, Prev)}}; - false -> false - end. - -changed(New, Prev) -> - Diff = emqx_utils_maps:diff_maps(New, Prev), - maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). - sorted_fold(Func, Conf) -> case lists:foldl(Func, [], to_sorted_list(Conf)) of [] -> ok; From 9b5c7065f2766f7790a6808a72a346c356f0f869 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 14 Jun 2023 09:12:20 +0800 Subject: [PATCH 3/3] feat: conf reload support --replace|merge --- apps/emqx_authn/src/emqx_authn.erl | 5 ++- apps/emqx_authz/src/emqx_authz.erl | 4 +++ apps/emqx_conf/src/emqx_conf_cli.erl | 53 +++++++++++++++++++--------- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index e717550f1..50287941e 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -26,7 +26,7 @@ get_enabled_authns/0 ]). --export([merge_config/1, import_config/1]). +-export([merge_config/1, merge_config_local/2, import_config/1]). -include("emqx_authn.hrl"). @@ -162,3 +162,6 @@ authn_list(Authn) when is_map(Authn) -> merge_config(AuthNs) -> emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}). + +merge_config_local(AuthNs, Opts) -> + emqx:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}, Opts). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 9fd6063e7..6f45a88b7 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -33,6 +33,7 @@ move/2, update/2, merge/1, + merge_local/2, authorize/5, %% for telemetry information get_enabled_authzs/0 @@ -127,6 +128,9 @@ lookup(Type) -> merge(NewConf) -> emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}). +merge_local(NewConf, Opts) -> + emqx:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}, Opts). + move(Type, ?CMD_MOVE_BEFORE(Before)) -> emqx_authz_utils:update_config( ?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))} diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 6035c1ec6..4231149d6 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -55,8 +55,12 @@ conf(["load", Path]) -> load_config(Path, merge); conf(["cluster_sync" | Args]) -> admins(Args); +conf(["reload", "--merge"]) -> + reload_etc_conf_on_local_node(merge); +conf(["reload", "--replace"]) -> + reload_etc_conf_on_local_node(replace); conf(["reload"]) -> - reload_etc_conf_on_local_node(); + conf(["reload", "--merge"]); conf(_) -> emqx_ctl:usage(usage_conf() ++ usage_sync()). @@ -98,7 +102,10 @@ admins(_) -> usage_conf() -> [ - {"conf reload", "reload etc/emqx.conf on local node"}, + {"conf reload --replace|--merge", "reload etc/emqx.conf on local node"}, + {"", "The new configuration values will be overlaid on the existing values by default."}, + {"", "use the --replace flag to replace existing values with the new ones instead."}, + {"----------------------------------", "------------"}, {"conf show_keys", "print all the currently used configuration keys."}, {"conf show []", "Print in-use configs (including default values) under the given key."}, @@ -181,7 +188,7 @@ load_config(Path, ReplaceOrMerge) -> case check_config(RawConf) of ok -> lists:foreach( - fun({K, V}) -> update_config(K, V, ReplaceOrMerge) end, + fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end, to_sorted_list(RawConf) ); {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> @@ -205,18 +212,34 @@ load_config(Path, ReplaceOrMerge) -> {error, bad_hocon_file} end. -update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> +update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> check_res(Key, emqx_authz:merge(Conf)); -update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> +update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> check_res(Key, emqx_authn:merge_config(Conf)); -update_config(Key, NewConf, merge) -> +update_config_cluster(Key, NewConf, merge) -> Merged = merge_conf(Key, NewConf), check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS)); -update_config(Key, Value, replace) -> +update_config_cluster(Key, Value, replace) -> check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)). -check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); -check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]). +-define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}). +update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> + check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS)); +update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, merge) -> + check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS)); +update_config_local(Key, NewConf, merge) -> + Merged = merge_conf(Key, NewConf), + check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS)); +update_config_local(Key, Value, replace) -> + check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS)). + +check_res(Key, Res) -> check_res(cluster, Key, Res). +check_res(Mode, Key, {ok, _} = Res) -> + emqx_ctl:print("load ~ts in ~p ok~n", [Key, Mode]), + Res; +check_res(_Mode, Key, {error, Reason} = Res) -> + emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]), + Res. check_config(Conf) -> case check_keys_is_not_readonly(Conf) of @@ -244,13 +267,13 @@ check_config_schema(Conf) -> sorted_fold(Fold, Conf). %% @doc Reload etc/emqx.conf to runtime config except for the readonly config --spec reload_etc_conf_on_local_node() -> ok | {error, term()}. -reload_etc_conf_on_local_node() -> +-spec reload_etc_conf_on_local_node(replace | merge) -> ok | {error, term()}. +reload_etc_conf_on_local_node(ReplaceOrMerge) -> case load_etc_config_file() of {ok, RawConf} -> case filter_readonly_config(RawConf) of {ok, Reloaded} -> - reload_config(Reloaded); + reload_config(Reloaded, ReplaceOrMerge); {error, Error} -> emqx_ctl:warning("check config failed~n~p~n", [Error]), {error, Error} @@ -301,14 +324,12 @@ filter_readonly_config(Raw) -> {error, Error} end. -reload_config(AllConf) -> +reload_config(AllConf, ReplaceOrMerge) -> Fold = fun({Key, Conf}, Acc) -> - case emqx:update_config([Key], Conf, #{persistent => false}) of + case update_config_local(Key, Conf, ReplaceOrMerge) of {ok, _} -> - emqx_ctl:print("Reloaded ~ts config ok~n", [Key]), Acc; Error -> - emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]), ?SLOG(error, #{ msg => "failed_to_reload_etc_config", key => Key,