From 180f5717659228a46617fa31f6f7666ce58ccd5f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 3 Apr 2023 09:12:23 +0800 Subject: [PATCH 01/13] feat: configuration priority ENV > emqx.conf > API --- apps/emqx/src/config/emqx_config_logger.erl | 23 +-- apps/emqx/src/emqx_config.erl | 189 ++++++++++++++++-- apps/emqx/src/emqx_config_handler.erl | 14 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 15 +- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 8 +- .../src/emqx_mgmt_api_configs.erl | 138 +++++-------- .../test/emqx_mgmt_api_configs_SUITE.erl | 53 +++-- 8 files changed, 289 insertions(+), 155 deletions(-) diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 15e4d3959..b0fc1ca67 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -32,25 +32,15 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?LOG), ok. -%% refresh logger config when booting, the override config may have changed after node start. +%% refresh logger config when booting, the cluster config may have changed after node start. %% Kernel's app env is confirmed before the node starts, -%% but we only copy cluster-override.conf from other node after this node starts, +%% but we only copy cluster.conf from other node after this node starts, %% so we need to refresh the logger config after this node starts. -%% It will not affect the logger config when cluster-override.conf is unchanged. +%% It will not affect the logger config when cluster.conf is unchanged. refresh_config() -> - Overrides = emqx_config:read_override_confs(), - refresh_config(Overrides). - -refresh_config(#{<<"log">> := _}) -> %% read the checked config LogConfig = emqx:get_config(?LOG, undefined), - Conf = #{log => LogConfig}, - ok = do_refresh_config(Conf); -refresh_config(_) -> - %% No config override found for 'log', do nothing - %% because the 'kernel' app should already be configured - %% from the base configs. i.e. emqx.conf + env vars - ok. + do_refresh_config(#{log => LogConfig}). %% this call is shared between initial config refresh at boot %% and dynamic config update from HTTP API @@ -61,10 +51,9 @@ do_refresh_config(Conf) -> ok = maybe_update_log_level(Level), ok. +%% always refresh config when the override config is changed post_config_update(?LOG, _Req, NewConf, _OldConf, _AppEnvs) -> - ok = do_refresh_config(#{log => NewConf}); -post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> - ok. + do_refresh_config(#{log => NewConf}). maybe_update_log_level(NewLevel) -> OldLevel = emqx_logger:get_primary_log_level(), diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bf3134568..41e38f184 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,17 +24,19 @@ init_load/2, init_load/3, read_override_conf/1, - read_override_confs/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, fill_defaults/2, fill_defaults/3, - save_configs/5, + save_configs/6, save_to_app_env/1, save_to_config_map/2, save_to_override_conf/2 ]). +-export([raw_conf_with_default/4]). +-export([remove_default_conf/2]). +-export([merge_envs/2]). -export([ get_root/1, @@ -329,23 +331,15 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(SchemaMod, parse_hocon(Conf), Opts); init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), + RootNames = get_root_names(), %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - Overrides = read_override_confs(), - RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), %% check configs against the schema {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), save_to_app_env(AppEnvs), ok = save_to_config_map(CheckedConf, RawConfAll). -%% @doc Read merged cluster + local overrides. -read_override_confs() -> - ClusterOverrides = read_override_conf(#{override_to => cluster}), - LocalOverrides = read_override_conf(#{override_to => local}), - hocon:deep_merge(ClusterOverrides, LocalOverrides). - %% keep the raw and non-raw conf has the same keys to make update raw conf easier. raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> Fun = fun(Name, Acc) -> @@ -375,6 +369,9 @@ schema_default(Schema) -> end. parse_hocon(Conf) -> + %% merge cluster-override.conf to local-override.conf + %% cluster-override.conf is deprecated, now is cluster.conf + merge_deprecated_cluster_override_to_local_override(), IncDirs = include_dirs(), case do_parse_hocon(Conf, IncDirs) of {ok, HoconMap} -> @@ -393,8 +390,12 @@ parse_hocon(Conf) -> do_parse_hocon(Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of - true -> hocon:binary(Conf, Opts); - false -> hocon:files(Conf, Opts) + true -> + hocon:binary(Conf, Opts); + false -> + LocalFile = override_conf_file(#{override_to => local}), + ClusterFile = override_conf_file(#{override_to => cluster}), + hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) end. include_dirs() -> @@ -483,6 +484,13 @@ read_override_conf(#{} = Opts) -> File = override_conf_file(Opts), load_hocon_file(File, map). +read_deprecated_override_conf() -> + ClusterFile = override_conf_file(#{override_to => cluster}), + DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"), + Conf = load_hocon_file(DeprecatedFile, map), + _ = file:delete(DeprecatedFile), + Conf. + override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of @@ -522,14 +530,54 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). --spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok. -save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) -> - %% We first try to save to override.conf, because saving to files is more error prone +-spec save_configs( + emqx_map_lib:config_key_path(), app_envs(), config(), raw_config(), raw_config(), update_opts() +) -> ok. + +save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> + Default = init_default(Paths0), + OverrideConf1 = remove_default_conf(OverrideConf, Default), + %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf, Opts), + ok = save_to_override_conf(OverrideConf1, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). +init_default(Paths0) -> + [Root | _] = [bin(Key) || Key <- Paths0], + SchemaMod = get_schema_mod(Root), + {_, {_, Schema}} = lists:keyfind(Root, 1, hocon_schema:roots(SchemaMod)), + fill_defaults(#{Root => schema_default(Schema)}). + +remove_default_conf(undefined, _) -> + undefined; +remove_default_conf(Conf, DefaultConf) when is_map(Conf) andalso is_map(DefaultConf) -> + maps:fold( + fun(Key, Value, Acc) -> + case maps:find(Key, DefaultConf) of + {ok, DefaultValue} -> + remove_default_conf(Value, DefaultValue, Key, Acc); + error -> + Acc + end + end, + Conf, + Conf + ). + +remove_default_conf(Value, Value, Key, Conf) -> + maps:remove(Key, Conf); +remove_default_conf(Value = #{}, DefaultValue = #{}, Key, Conf) -> + case remove_default_conf(Value, DefaultValue) of + SubValue when SubValue =:= #{} -> maps:remove(Key, Conf); + SubValue -> maps:put(Key, SubValue, Conf) + end; +remove_default_conf(Value, DefaultValue, Key, Conf) -> + case try_bin(DefaultValue) =:= try_bin(Value) of + true -> maps:remove(Key, Conf); + false -> Conf + end. + %% we ignore kernel app env, %% because the old app env will be used in emqx_config_logger:post_config_update/5 -define(IGNORE_APPS, [kernel]). @@ -678,6 +726,16 @@ atom(Str) when is_list(Str) -> atom(Atom) when is_atom(Atom) -> Atom. +try_bin(Bin) when is_binary(Bin) -> Bin; +try_bin([Bin | _] = List) when is_binary(Bin) -> List; +try_bin([Atom | _] = List) when is_atom(Atom) -> [atom_to_binary(A) || A <- List]; +try_bin([Map | _] = Maps) when is_map(Map) -> Maps; +try_bin(Str) when is_list(Str) -> list_to_binary(Str); +try_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +try_bin(Int) when is_integer(Int) -> integer_to_binary(Int); +try_bin(Float) when is_float(Float) -> float_to_binary(Float); +try_bin(Term) -> Term. + bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). @@ -699,3 +757,98 @@ atom_conf_path(Path, ExpFun, OnFail) -> error(Err) end end. + +merge_deprecated_cluster_override_to_local_override() -> + case read_deprecated_override_conf() of + DeprecatedConf when DeprecatedConf =/= #{} -> + LocalOverrides = read_override_conf(#{override_to => local}), + MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf), + _ = save_to_override_conf(MergedConf, #{override_to => local}), + ok; + _ -> + ok + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +remove_default_conf_test() -> + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => 100}) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => <<"100">>}) + ), + ?assertEqual( + #{<<"def">> => 100}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => #{<<"bar">> => 100}}) + ), + ?assertEqual( + #{<<"def">> => #{<<"edf">> => 321}}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ + <<"def">> => #{<<"abc">> => 100, <<"edf">> => 123} + }) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ + <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => 321} + }) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => <<"true">>}}, #{ + <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => true} + }) + ), + ?assertEqual( + #{}, + remove_default_conf( + #{ + <<"bytes_in">> => + #{ + <<"capacity">> => infinity, + <<"initial">> => 0, + <<"rate">> => infinity + } + }, + #{ + <<"bytes_in">> => + #{ + <<"capacity">> => <<"infinity">>, + <<"initial">> => <<"0">>, + <<"rate">> => <<"infinity">> + } + } + ) + ), + ?assertEqual( + #{}, + remove_default_conf( + #{ + <<"limiter">> => #{ + <<"connection">> => + #{ + <<"capacity">> => 1000, + <<"initial">> => <<"0">>, + <<"rate">> => <<"1000/s">> + } + } + }, + #{ + <<"limiter">> => #{ + <<"connection">> => + #{ + <<"capacity">> => <<"1000">>, + <<"initial">> => <<"0">>, + <<"rate">> => <<"1000/s">> + } + } + } + ) + ), + ok. + +-endif. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a0a99b62e..ee22c297e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -271,8 +271,10 @@ do_update_config( SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf), SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of - {ok, NewUpdateReq} -> merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); - Error -> Error + {ok, NewUpdateReq} -> + merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); + Error -> + Error end. check_and_save_configs( @@ -289,7 +291,9 @@ check_and_save_configs( OldConf = emqx_config:get_root(ConfKeyPath), case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of {ok, Result0} -> - ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts), + ok = emqx_config:save_configs( + ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, Opts + ), Result1 = return_change_result(ConfKeyPath, UpdateArgs), {ok, Result1#{post_config_update => Result0}}; Error -> @@ -560,11 +564,11 @@ check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> allow; {error, Error} -> ?SLOG(error, #{ - msg => "prevent_remove_local_override_conf", + msg => "prevent_remove_local_conf", config_key_path => ConfKeyPath, error => Error }), - {deny, "Disable changed from local-override.conf"} + {deny, "Disable changed from local conf"} end; {not_found, _, _} -> allow diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 8126b35c6..6e2dbfef1 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -21,8 +21,8 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(LOCAL_CONF, "/tmp/local-override.conf"). --define(CLUSTER_CONF, "/tmp/cluster-override.conf"). +-define(LOCAL_CONF, "/tmp/local.conf"). +-define(CLUSTER_CONF, "/tmp/cluster.conf"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -221,12 +221,6 @@ t_local_override_update_remove(_Config) -> ), ?assertMatch(0.7, emqx:get_config(KeyPath)), - KeyPath2 = [sysmon, os, cpu_low_watermark], - ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE), - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts) - ), - %% remove ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)), ?assertEqual( @@ -251,7 +245,6 @@ t_local_override_update_remove(_Config) -> ?assert(length(OSKey1) > 1), ok = emqx_config_handler:remove_handler(KeyPath), - ok = emqx_config_handler:remove_handler(KeyPath2), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), ok. @@ -426,9 +419,9 @@ wait_for_new_pid() -> callback_error(FailedPath, Update, Error) -> Opts = #{rawconf_with_defaults => true}, ok = emqx_config_handler:add_handler(FailedPath, ?MODULE), - Old = emqx:get_raw_config(FailedPath), + Old = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)), - New = emqx:get_raw_config(FailedPath), + New = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Old, New), ok = emqx_config_handler:remove_handler(FailedPath), ok. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index f689997c2..ed246b4ae 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1076,10 +1076,10 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster-override.conf"). + tr_override_conf_file(Conf, "cluster.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local-override.conf"). + tr_override_conf_file(Conf, "local.conf"). tr_override_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index c34eb9dc3..eb67490bc 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -85,7 +85,7 @@ create_data_dir() -> ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). set_data_dir_env() -> Node = atom_to_list(node()), @@ -100,14 +100,14 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), - application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), + application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster.hocon"), ok. assert_data_copy_done([First0 | Rest]) -> First = atom_to_list(First0), {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), - {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster-override.conf"), + {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), lists:foreach( fun(Node0) -> Node = atom_to_list(Node0), @@ -118,7 +118,7 @@ assert_data_copy_done([First0 | Rest]) -> ), ?assertEqual( {ok, FakeOverrideFile}, - file:read_file(Node ++ "/configs/cluster-override.conf"), + file:read_file(Node ++ "/configs/cluster.hocon"), #{node => Node} ), ?assertEqual( diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..56572b41e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -31,41 +31,23 @@ global_zone_configs/3 ]). --export([gen_schema/1]). - -define(PREFIX, "/configs/"). -define(PREFIX_RESET, "/configs_reset/"). -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). -define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}). -define(TAGS, ["Configs"]). --define(EXCLUDES, - [ - <<"exhook">>, - <<"gateway">>, - <<"plugins">>, - <<"bridges">>, - <<"rule_engine">>, - <<"authorization">>, - <<"authentication">>, - <<"rpc">>, - <<"connectors">>, - <<"slow_subs">>, - <<"psk_authentication">>, - <<"topic_metrics">>, - <<"rewrite">>, - <<"auto_subscribe">>, - <<"retainer">>, - <<"statsd">>, - <<"delayed">>, - <<"event_message">>, - <<"prometheus">>, - <<"telemetry">>, - <<"listeners">>, - <<"license">>, - <<"api_key">> - ] ++ global_zone_roots() -). +-define(ROOT_KEYS, [ + <<"dashboard">>, + <<"alarm">>, + <<"sys_topics">>, + <<"sysmon">>, + <<"limiter">>, + <<"trace">>, + <<"log">>, + <<"persistent_session_store">>, + <<"zones">> +]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -119,7 +101,6 @@ schema("/configs_reset/:rootname") -> "- For a config entry that has default value, this resets it to the default value;\n" "- For a config entry that has no default value, an error 400 will be returned" >>, - summary => <<"Reset config entry">>, %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. @@ -214,12 +195,11 @@ fields(Field) -> %%%============================================================================================== %% HTTP API Callbacks config(get, _Params, Req) -> + [Path] = conf_path(Req), + {200, get_raw_config(Path)}; +config(put, #{body := NewConf}, Req) -> Path = conf_path(Req), - {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()), - {200, Conf}; -config(put, #{body := Body}, Req) -> - Path = conf_path(Req), - case emqx_conf:update(Path, Body, ?OPTS) of + case emqx_conf:update(Path, NewConf, ?OPTS) of {ok, #{raw_config := RawConf}} -> {200, RawConf}; {error, {permission_denied, Reason}} -> @@ -229,28 +209,29 @@ config(put, #{body := Body}, Req) -> end. global_zone_configs(get, _Params, _Req) -> - Paths = global_zone_roots(), - Zones = lists:foldl( - fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end, - #{}, - Paths - ), - {200, Zones}; + {200, get_zones()}; global_zone_configs(put, #{body := Body}, _Req) -> + PrevZones = get_zones(), Res = maps:fold( fun(Path, Value, Acc) -> - case emqx_conf:update([Path], Value, ?OPTS) of - {ok, #{raw_config := RawConf}} -> - Acc#{Path => RawConf}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update global zone failed", - reason => Reason, - path => Path, - value => Value - }), - Acc + PrevValue = maps:get(Path, PrevZones), + case Value =/= PrevValue of + true -> + case emqx_conf:update([Path], Value, ?OPTS) of + {ok, #{raw_config := RawConf}} -> + Acc#{Path => RawConf}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update global zone failed", + reason => Reason, + path => Path, + value => Value + }), + Acc + end; + false -> + Acc#{Path => Value} end end, #{}, @@ -298,13 +279,30 @@ conf_path_reset(Req) -> get_full_config() -> emqx_config:fill_defaults( - maps:without( - ?EXCLUDES, + maps:with( + ?ROOT_KEYS, emqx:get_raw_config([]) ), #{obfuscate_sensitive_values => true} ). +get_raw_config(Path) -> + #{Path := Conf} = + emqx_config:fill_defaults( + #{Path => emqx:get_raw_config([Path])}, + #{obfuscate_sensitive_values => true} + ), + Conf. + +get_zones() -> + lists:foldl( + fun(Path, Acc) -> + maps:merge(Acc, get_config_with_default(Path)) + end, + #{}, + global_zone_roots() + ). + get_config_with_default(Path) -> emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}). @@ -317,40 +315,12 @@ conf_path_from_querystr(Req) -> config_list() -> Mod = emqx_conf:schema_module(), Roots = hocon_schema:roots(Mod), - lists:foldl(fun(Key, Acc) -> lists:keydelete(Key, 1, Acc) end, Roots, ?EXCLUDES). + lists:foldl(fun(Key, Acc) -> [lists:keyfind(Key, 1, Roots) | Acc] end, [], ?ROOT_KEYS). conf_path(Req) -> <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). -%% TODO: generate from hocon schema -gen_schema(Conf) when is_boolean(Conf) -> - with_default_value(#{type => boolean}, Conf); -gen_schema(Conf) when is_binary(Conf); is_atom(Conf) -> - with_default_value(#{type => string}, Conf); -gen_schema(Conf) when is_number(Conf) -> - with_default_value(#{type => number}, Conf); -gen_schema(Conf) when is_list(Conf) -> - case io_lib:printable_unicode_list(Conf) of - true -> - gen_schema(unicode:characters_to_binary(Conf)); - false -> - #{type => array, items => gen_schema(hd(Conf))} - end; -gen_schema(Conf) when is_map(Conf) -> - #{ - type => object, - properties => - maps:map(fun(_K, V) -> gen_schema(V) end, Conf) - }; -gen_schema(_Conf) -> - %% the conf is not of JSON supported type, it may have been converted - %% by the hocon schema - #{type => string}. - -with_default_value(Type, Value) -> - Type#{example => emqx_map_lib:binary_string(Value)}. - global_zone_roots() -> lists:map(fun({K, _}) -> K end, global_zone_schema()). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..f5ee4e5be 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -55,12 +55,17 @@ t_update(_Config) -> %% update ok {ok, SysMon} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon, - NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), + NewSysMon = #{<<"vm">> => #{<<"busy_port">> => not BusyPort}}, {ok, #{}} = update_config(<<"sysmon">>, NewSysMon), {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, ?assertEqual(BusyPort, not BusyPort1), assert_busy_port(BusyPort1), + %% Make sure the override config is updated, and remove the default value. + ?assertEqual( + #{<<"vm">> => #{<<"busy_port">> => BusyPort1}}, + maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster})) + ), %% update failed ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), @@ -130,6 +135,8 @@ t_global_zone(_Config) -> NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), + %% Make sure the override config is updated, and remove the default value. + ?assertEqual(#{<<"max_qos_allowed">> => 1}, read_conf(<<"mqtt">>)), BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), @@ -145,6 +152,10 @@ t_global_zone(_Config) -> %% the default value is 2 ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), + + DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), + {ok, #{}} = update_global_zone(DefaultZones), + ?assertEqual(undefined, read_conf(<<"mqtt">>)), ok. get_global_zone() -> @@ -169,7 +180,7 @@ t_dashboard(_Config) -> Https1 = #{enable => true, bind => 18084}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, - update_config("dashboard", Dashboard#{<<"https">> => Https1}) + update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}}) ), Https2 = #{ @@ -179,35 +190,41 @@ t_dashboard(_Config) -> cacertfile => "etc/certs/badcacert.pem", certfile => "etc/certs/badcert.pem" }, - Dashboard2 = Dashboard#{listeners => Listeners#{https => Https2}}, + Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config("dashboard", Dashboard2) ), - Keyfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), - Certfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), - Cacertfile = emqx_common_test_helpers:app_path( + KeyFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), + CertFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), + CacertFile = emqx_common_test_helpers:app_path( emqx, filename:join(["etc", "certs", "cacert.pem"]) ), Https3 = #{ - enable => true, - bind => 18084, - keyfile => Keyfile, - cacertfile => Cacertfile, - certfile => Certfile + <<"enable">> => true, + <<"bind">> => 18084, + <<"keyfile">> => list_to_binary(KeyFile), + <<"cacertfile">> => list_to_binary(CacertFile), + <<"certfile">> => list_to_binary(CertFile) }, - Dashboard3 = Dashboard#{listeners => Listeners#{https => Https3}}, + Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)), - Dashboard4 = Dashboard#{listeners => Listeners#{https => #{enable => false}}}, + Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"enable">> => false}}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)), + {ok, Dashboard41} = get_config("dashboard"), + ?assertEqual( + Https3#{<<"enable">> => false}, + read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]), + Dashboard41 + ), ?assertMatch({ok, _}, update_config("dashboard", Dashboard)), {ok, Dashboard1} = get_config("dashboard"), ?assertNotEqual(Dashboard, Dashboard1), - timer:sleep(1000), + timer:sleep(1500), ok. t_configs_node({'init', Config}) -> @@ -288,3 +305,11 @@ reset_config(Name, Key) -> {ok, []} -> ok; Error -> Error end. + +read_conf(RootKeys) when is_list(RootKeys) -> + case emqx_config:read_override_conf(#{override_to => cluster}) of + undefined -> undefined; + Conf -> emqx_map_lib:deep_get(RootKeys, Conf, undefined) + end; +read_conf(RootKey) -> + read_conf([RootKey]). From 5f4ea3b6d8621ac667b7af121fac452617cea6c7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 7 Apr 2023 09:17:54 +0800 Subject: [PATCH 02/13] feat: deprecated cluster-override.conf --- apps/emqx/src/emqx_config.erl | 142 ++++++++++++++++-------- apps/emqx_conf/src/emqx_conf_app.erl | 29 +++-- apps/emqx_conf/src/emqx_conf_schema.erl | 16 ++- 3 files changed, 127 insertions(+), 60 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 41e38f184..68a8563d8 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,6 +24,7 @@ init_load/2, init_load/3, read_override_conf/1, + has_deprecated_conf/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, @@ -32,7 +33,7 @@ save_configs/6, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/2 + save_to_override_conf/3 ]). -export([raw_conf_with_default/4]). -export([remove_default_conf/2]). @@ -331,14 +332,35 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(SchemaMod, parse_hocon(Conf), Opts); init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - RootNames = get_root_names(), - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll). + case has_deprecated_conf() of + false -> + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); + true -> + %% deprecated conf will be removed in 5.1 + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + Overrides = read_override_confs(), + RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), + RootNames = get_root_names(), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll) + end. + +%% @doc Read merged cluster + local overrides. +read_override_confs() -> + ClusterOverrides = read_override_conf(#{override_to => cluster}), + LocalOverrides = read_override_conf(#{override_to => local}), + hocon:deep_merge(ClusterOverrides, LocalOverrides). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> @@ -369,11 +391,9 @@ schema_default(Schema) -> end. parse_hocon(Conf) -> - %% merge cluster-override.conf to local-override.conf - %% cluster-override.conf is deprecated, now is cluster.conf - merge_deprecated_cluster_override_to_local_override(), + HasDeprecatedConf = has_deprecated_conf(), IncDirs = include_dirs(), - case do_parse_hocon(Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedConf, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> @@ -387,14 +407,20 @@ parse_hocon(Conf) -> error(failed_to_load_hocon_conf) end. -do_parse_hocon(Conf, IncDirs) -> +do_parse_hocon(true, Conf, IncDirs) -> + Opts = #{format => map, include_dirs => IncDirs}, + case is_binary(Conf) of + true -> hocon:binary(Conf, Opts); + false -> hocon:files(Conf, Opts) + end; +do_parse_hocon(false, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of true -> hocon:binary(Conf, Opts); false -> - LocalFile = override_conf_file(#{override_to => local}), - ClusterFile = override_conf_file(#{override_to => cluster}), + LocalFile = deprecated_override_conf_file(#{override_to => local}), + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) end. @@ -467,10 +493,14 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = override_conf_file(#{override_to => local}), - F2 = override_conf_file(#{override_to => cluster}), + F1 = deprecated_override_conf_file(#{override_to => local}), + F2 = deprecated_override_conf_file(#{override_to => cluster}), + F3 = conf_file(#{override_to => local}), + F4 = conf_file(#{override_to => cluster}), ok = ensure_file_deleted(F1), - ok = ensure_file_deleted(F2). + ok = ensure_file_deleted(F2), + ok = ensure_file_deleted(F3), + ok = ensure_file_deleted(F4). ensure_file_deleted(F) -> case file:delete(F) of @@ -481,24 +511,39 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> - File = override_conf_file(Opts), + File = + case has_deprecated_conf() of + true -> deprecated_override_conf_file(Opts); + false -> conf_file(Opts) + end, load_hocon_file(File, map). -read_deprecated_override_conf() -> - ClusterFile = override_conf_file(#{override_to => cluster}), - DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"), - Conf = load_hocon_file(DeprecatedFile, map), - _ = file:delete(DeprecatedFile), - Conf. +has_deprecated_conf() -> + DeprecatedFile = deprecated_cluster_conf_file(), + filelib:is_regular(DeprecatedFile). -override_conf_file(Opts) when is_map(Opts) -> +deprecated_cluster_conf_file() -> + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), + filename:join(filename:dirname(ClusterFile), "cluster-override.conf"). + +deprecated_override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -override_conf_file(Which) when is_atom(Which) -> +deprecated_override_conf_file(Which) when is_atom(Which) -> + application:get_env(emqx, Which, undefined). + +conf_file(Opts) when is_map(Opts) -> + Key = + case maps:get(override_to, Opts, cluster) of + local -> local_conf_file; + cluster -> cluster_conf_file + end, + application:get_env(emqx, Key, undefined); +conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. @@ -539,7 +584,8 @@ save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> OverrideConf1 = remove_default_conf(OverrideConf, Default), %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf1, Opts), + HasDeprecated = has_deprecated_conf(), + ok = save_to_override_conf(HasDeprecated, OverrideConf1, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). @@ -592,11 +638,12 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put(Conf), ?MODULE:put_raw(RawConf). --spec save_to_override_conf(raw_config(), update_opts()) -> ok | {error, term()}. -save_to_override_conf(undefined, _) -> +-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}. +save_to_override_conf(_, undefined, _) -> ok; -save_to_override_conf(RawConf, Opts) -> - case override_conf_file(Opts) of +%% TODO: Remove deprecated override conf file when 5.1 +save_to_override_conf(true, RawConf, Opts) -> + case deprecated_override_conf_file(Opts) of undefined -> ok; FileName -> @@ -612,6 +659,24 @@ save_to_override_conf(RawConf, Opts) -> }), {error, Reason} end + end; +save_to_override_conf(false, RawConf, Opts) -> + case conf_file(Opts) of + undefined -> + ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_save_conf_file", + filename => FileName, + reason => Reason + }), + {error, Reason} + end end. add_handlers() -> @@ -758,17 +823,6 @@ atom_conf_path(Path, ExpFun, OnFail) -> end end. -merge_deprecated_cluster_override_to_local_override() -> - case read_deprecated_override_conf() of - DeprecatedConf when DeprecatedConf =/= #{} -> - LocalOverrides = read_override_conf(#{override_to => local}), - MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf), - _ = save_to_override_conf(MergedConf, #{override_to => local}), - ok; - _ -> - ok - end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 51fc5c2e2..45631619b 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,7 +60,14 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - #{wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node} + HasDeprecateConf = emqx_config:has_deprecated_conf(), + #{ + wall_clock => WallClock, + conf => Conf, + tnx_id => TnxId, + node => Node, + has_deprecated_conf => HasDeprecateConf + } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of {atomic, Res} -> {ok, Res}; @@ -153,10 +160,10 @@ copy_override_conf_from_core_node() -> {ok, ?DEFAULT_INIT_TXN_ID}; false -> %% retry in some time - Jitter = rand:uniform(2_000), - Timeout = 10_000 + Jitter, + Jitter = rand:uniform(2000), + Timeout = 10000 + Jitter, ?SLOG(info, #{ - msg => "copy_override_conf_from_core_node_retry", + msg => "copy_cluster_conf_from_core_node_retry", timeout => Timeout, nodes => Nodes, failed => Failed, @@ -168,18 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, + HasDeprecatedConf = maps:get(has_deprecated_conf, Info, false), ?SLOG(debug, #{ - msg => "copy_override_conf_from_core_node_success", + msg => "copy_cluster_conf_from_core_node_success", node => Node, - cluster_override_conf_file => application:get_env( - emqx, cluster_override_conf_file - ), - local_override_conf_file => application:get_env( - emqx, local_override_conf_file - ), - data_dir => emqx:data_dir() + has_deprecated_conf => HasDeprecatedConf, + data_dir => emqx:data_dir(), + tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( + HasDeprecatedConf, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ed246b4ae..a67d2a100 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1028,7 +1028,9 @@ translation("emqx") -> [ {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, - {"local_override_conf_file", fun tr_local_override_conf_file/1} + {"local_override_conf_file", fun tr_local_override_conf_file/1}, + {"cluster_conf_file", fun tr_cluster_conf_file/1}, + {"local_conf_file", fun tr_local_conf_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1076,12 +1078,18 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster.conf"). + tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local.conf"). + tr_conf_file(Conf, "local-overide.conf"). -tr_override_conf_file(Conf, Filename) -> +tr_cluster_conf_file(Conf) -> + tr_conf_file(Conf, "cluster.hocon"). + +tr_local_conf_file(Conf) -> + tr_conf_file(Conf, "local.hocon"). + +tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable [_ | _] = DataDir, From a271ba5ff9ec53741a9f3ac85dd46fe6bc9e4bae Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 10 Apr 2023 14:52:49 +0800 Subject: [PATCH 03/13] feat: don't check_permission on local.conf --- apps/emqx/src/emqx_config_handler.erl | 117 +------------------ apps/emqx/test/emqx_config_handler_SUITE.erl | 49 -------- 2 files changed, 5 insertions(+), 161 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index ee22c297e..3e18a172e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -43,7 +43,6 @@ terminate/2, code_change/3 ]). --export([is_mutable/3]). -define(MOD, {mod}). -define(WKEY, '?'). @@ -230,26 +229,15 @@ process_update_request([_], _Handlers, {remove, _Opts}) -> process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of - allow -> - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), - OverrideConf = remove_from_override_config(BinKeyPath, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + OverrideConf = remove_from_override_config(BinKeyPath, Opts), + {ok, NewRawConf, OverrideConf, Opts}; process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> - BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(update, BinKeyPath, NewRawConf, Opts) of - allow -> - OverrideConf = merge_to_override_config(NewRawConf, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + OverrideConf = merge_to_override_config(NewRawConf, Opts), + {ok, NewRawConf, OverrideConf, Opts}; Error -> Error end. @@ -550,98 +538,3 @@ load_prev_handlers() -> save_handlers(Handlers) -> application:set_env(emqx, ?MODULE, Handlers). - -check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) -> - allow; -check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> - case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of - {ok, NewRaw} -> - LocalOverride = emqx_config:read_override_conf(#{override_to => local}), - case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of - {ok, LocalRaw} -> - case is_mutable(Action, NewRaw, LocalRaw) of - ok -> - allow; - {error, Error} -> - ?SLOG(error, #{ - msg => "prevent_remove_local_conf", - config_key_path => ConfKeyPath, - error => Error - }), - {deny, "Disable changed from local conf"} - end; - {not_found, _, _} -> - allow - end; - {not_found, _, _} -> - allow - end. - -is_mutable(Action, NewRaw, LocalRaw) -> - try - KeyPath = [], - is_mutable(KeyPath, Action, NewRaw, LocalRaw) - catch - throw:Error -> Error - end. - --define(REMOVE_FAILED, "remove_failed"). --define(UPDATE_FAILED, "update_failed"). - -is_mutable(KeyPath, Action, New = #{}, Local = #{}) -> - maps:foreach( - fun(Key, SubLocal) -> - case maps:find(Key, New) of - error -> ok; - {ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal) - end - end, - Local - ); -is_mutable(KeyPath, remove, Update, Origin) -> - throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}}); -is_mutable(_KeyPath, update, Val, Val) -> - ok; -is_mutable(KeyPath, update, Update, Origin) -> - throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -is_mutable_update_test() -> - Action = update, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, d], 2, 3}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - -is_mutable_remove_test() -> - Action = remove, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})), - - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - --endif. diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 6e2dbfef1..cea956825 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -200,55 +200,6 @@ t_sub_key_update_remove(_Config) -> ok = emqx_config_handler:remove_handler(KeyPath2), ok. -t_local_override_update_remove(_Config) -> - application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF), - application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF), - KeyPath = [sysmon, os, cpu_high_watermark], - ok = emqx_config_handler:add_handler(KeyPath, ?MODULE), - LocalOpts = #{override_to => local}, - {ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts), - ?assertMatch( - #{ - config := 0.7, - post_config_update := #{}, - raw_config := <<"70%">> - }, - Res - ), - ClusterOpts = #{override_to => cluster}, - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts) - ), - ?assertMatch(0.7, emqx:get_config(KeyPath)), - - %% remove - ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath, #{override_to => local}) - ), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath) - ), - ?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)), - OSKey = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)), - ?assert(length(OSKey) > 0), - - ?assertEqual( - {ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}}, - emqx:reset_config(KeyPath, ClusterOpts) - ), - OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)), - ?assert(length(OSKey1) > 1), - - ok = emqx_config_handler:remove_handler(KeyPath), - application:unset_env(emqx, local_override_conf_file), - application:unset_env(emqx, cluster_override_conf_file), - ok. - t_check_failed(_Config) -> KeyPath = [sysmon, os, cpu_check_interval], Opts = #{rawconf_with_defaults => true}, From 7f870257b2e29484e82d0281ebc3ffa9eb9e8fe9 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 10 Apr 2023 17:26:17 +0800 Subject: [PATCH 04/13] test: fix failed ct --- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index eb67490bc..e6b15ad13 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -100,7 +100,7 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), - application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), ok. assert_data_copy_done([First0 | Rest]) -> @@ -108,6 +108,7 @@ assert_data_copy_done([First0 | Rest]) -> {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), + {ok, ExpectFake} = hocon:binary(FakeOverrideFile), lists:foreach( fun(Node0) -> Node = atom_to_list(Node0), @@ -117,8 +118,8 @@ assert_data_copy_done([First0 | Rest]) -> #{node => Node} ), ?assertEqual( - {ok, FakeOverrideFile}, - file:read_file(Node ++ "/configs/cluster.hocon"), + {ok, ExpectFake}, + hocon:files([Node ++ "/configs/cluster.hocon"]), #{node => Node} ), ?assertEqual( From fa753cf333664b04d0a1ed60c68328027a43f477 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 14:25:44 +0800 Subject: [PATCH 05/13] fix: failed ct test --- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 53 +++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index e6b15ad13..21e22519e 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -27,7 +27,7 @@ all() -> t_copy_conf_override_on_restarts(_Config) -> ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([core, core, core]), + Cluster = cluster([cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})]), %% 1. Start all nodes Nodes = start_cluster(Cluster), @@ -41,7 +41,7 @@ t_copy_conf_override_on_restarts(_Config) -> %% crash and eventually all nodes should be ready. start_cluster_async(Cluster), - timer:sleep(15_000), + timer:sleep(15000), assert_config_load_done(Nodes), @@ -50,11 +50,11 @@ t_copy_conf_override_on_restarts(_Config) -> stop_cluster(Nodes) end. -t_copy_data_dir(_Config) -> +t_copy_new_data_dir(_Config) -> net_kernel:start(['master1@127.0.0.1', longnames]), ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([{core, copy1}, {core, copy2}, {core, copy3}]), + Cluster = cluster([cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})]), %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), @@ -66,7 +66,30 @@ t_copy_data_dir(_Config) -> ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes), + assert_data_copy_done(Nodes, "/configs/cluster.hocon"), + stop_cluster(Nodes), + ok + after + stop_cluster(Nodes) + end. + +t_copy_deprecated_data_dir(_Config) -> + net_kernel:start(['master2@127.0.0.1', longnames]), + ct:timetrap({seconds, 120}), + snabbkaffe:fix_ct_logging(), + Cluster = cluster([cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})]), + + %% 1. Start all nodes + [First | Rest] = Nodes = start_cluster(Cluster), + try + assert_config_load_done(Nodes), + rpc:call(First, ?MODULE, create_deprecated_data_dir, []), + {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), + {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), + ok = rpc:call(First, application, start, [emqx_conf]), + {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), + + assert_data_copy_done(Nodes, "/configs/cluster-override.conf"), stop_cluster(Nodes), ok after @@ -87,6 +110,16 @@ create_data_dir() -> Telemetry = <<"telemetry.enable = false">>, ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). +create_deprecated_data_dir() -> + Node = atom_to_list(node()), + ok = filelib:ensure_dir(Node ++ "/certs/"), + ok = filelib:ensure_dir(Node ++ "/authz/"), + ok = filelib:ensure_dir(Node ++ "/configs/"), + ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), + ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), + Telemetry = <<"telemetry.enable = false">>, + ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + set_data_dir_env() -> Node = atom_to_list(node()), %% will create certs and authz dir @@ -101,13 +134,14 @@ set_data_dir_env() -> application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. -assert_data_copy_done([First0 | Rest]) -> +assert_data_copy_done([First0 | Rest], File) -> First = atom_to_list(First0), {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), - {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), + {ok, FakeOverrideFile} = file:read_file(First ++ File), {ok, ExpectFake} = hocon:binary(FakeOverrideFile), lists:foreach( fun(Node0) -> @@ -119,7 +153,7 @@ assert_data_copy_done([First0 | Rest]) -> ), ?assertEqual( {ok, ExpectFake}, - hocon:files([Node ++ "/configs/cluster.hocon"]), + hocon:files([Node ++ File]), #{node => Node} ), ?assertEqual( @@ -174,3 +208,6 @@ cluster(Specs) -> ok end} ]). + +cluster_spec({Type, Num}) -> + {Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}. From c21744c2601cd8607b71455bc4441037ae67de9c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 17:09:24 +0800 Subject: [PATCH 06/13] fix: hocon_pp crash when atom_key --- apps/emqx/rebar.config | 2 +- apps/emqx/test/emqx_authentication_SUITE.erl | 8 ++++++-- apps/emqx/test/emqx_common_test_helpers.erl | 7 +++++++ mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9079322eb..bb89dfe52 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 2c83162ed..0190ab936 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -95,13 +95,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + LogLevel = emqx_logger:get_primary_log_level(), + ok = emqx_logger:set_log_level(debug), application:set_env(ekka, strict_mode, true), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), - Config. + [{log_level, LogLevel} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_common_test_helpers:stop_apps([]), + LogLevel = ?config(log_level), + emqx_logger:set_log_level(LogLevel), ok. init_per_testcase(Case, Config) -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 406183094..bc9da2a9e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -314,6 +314,8 @@ stop_apps(Apps) -> ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), + application:unset_env(emqx, local_conf_file), + application:unset_env(emqx, cluster_conf_file), application:unset_env(gen_rpc, port_discovery), ok. @@ -462,6 +464,11 @@ force_set_config_file_paths(emqx_conf, [Path] = Paths) -> ok = file:write_file(Path, Bin, [append]), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(emqx, Paths) -> + %% we need init cluster conf, so we can save the cluster conf to the file + application:set_env(emqx, local_override_conf_file, "local_override.conf"), + application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"), + application:set_env(emqx, local_conf_file, "local.hocon"), + application:set_env(emqx, cluster_conf_file, "cluster.hocon"), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(_, _) -> ok. diff --git a/mix.exs b/mix.exs index 582860039..55036d498 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.38.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.38.1", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 9b67b6cce..7e783b56d 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From c0e6e79bcddca8eef85dd8a10081f1608789f29f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 20:51:40 +0800 Subject: [PATCH 07/13] feat: don't remove default value when save config --- apps/emqx/src/emqx_config.erl | 250 +++++-------------- apps/emqx/src/emqx_config_handler.erl | 4 +- apps/emqx/test/emqx_common_test_helpers.erl | 4 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 2 - apps/emqx_conf/src/emqx_conf_app.erl | 10 +- apps/emqx_conf/src/emqx_conf_schema.erl | 10 +- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 1 + 7 files changed, 67 insertions(+), 214 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 68a8563d8..75396d779 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,19 +24,18 @@ init_load/2, init_load/3, read_override_conf/1, - has_deprecated_conf/0, + has_deprecated_file/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, fill_defaults/2, fill_defaults/3, - save_configs/6, + save_configs/5, save_to_app_env/1, save_to_config_map/2, save_to_override_conf/3 ]). -export([raw_conf_with_default/4]). --export([remove_default_conf/2]). -export([merge_envs/2]). -export([ @@ -329,32 +328,32 @@ init_load(SchemaMod, ConfFiles) -> %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> - init_load(SchemaMod, parse_hocon(Conf), Opts); -init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> + HasDeprecatedFile = has_deprecated_file(), + RawConf = parse_hocon(HasDeprecatedFile, Conf), + init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts). + +init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> + %% deprecated conf will be removed in 5.1 + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + Overrides = read_override_confs(), + RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), + RootNames = get_root_names(), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); +init_load(false, SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - case has_deprecated_conf() of - false -> - RootNames = get_root_names(), - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll); - true -> - %% deprecated conf will be removed in 5.1 - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - Overrides = read_override_confs(), - RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll) - end. + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll). %% @doc Read merged cluster + local overrides. read_override_confs() -> @@ -390,21 +389,20 @@ schema_default(Schema) -> #{} end. -parse_hocon(Conf) -> - HasDeprecatedConf = has_deprecated_conf(), +parse_hocon(HasDeprecatedFile, Conf) -> IncDirs = include_dirs(), - case do_parse_hocon(HasDeprecatedConf, Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedFile, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> ?SLOG(error, #{ - msg => "failed_to_load_hocon_conf", + msg => "failed_to_load_hocon_file", reason => Reason, pwd => file:get_cwd(), include_dirs => IncDirs, config_file => Conf }), - error(failed_to_load_hocon_conf) + error(failed_to_load_hocon_file) end. do_parse_hocon(true, Conf, IncDirs) -> @@ -416,12 +414,12 @@ do_parse_hocon(true, Conf, IncDirs) -> do_parse_hocon(false, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of + %% only use in test true -> hocon:binary(Conf, Opts); false -> - LocalFile = deprecated_override_conf_file(#{override_to => local}), - ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), - hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) + ClusterFile = cluster_hocon_file(#{override_to => cluster}), + hocon:files([ClusterFile | Conf], Opts) end. include_dirs() -> @@ -493,14 +491,12 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = deprecated_override_conf_file(#{override_to => local}), - F2 = deprecated_override_conf_file(#{override_to => cluster}), - F3 = conf_file(#{override_to => local}), - F4 = conf_file(#{override_to => cluster}), + F1 = deprecated_conf_file(#{override_to => local}), + F2 = deprecated_conf_file(#{override_to => cluster}), + F3 = cluster_hocon_file(#{override_to => cluster}), ok = ensure_file_deleted(F1), ok = ensure_file_deleted(F2), - ok = ensure_file_deleted(F3), - ok = ensure_file_deleted(F4). + ok = ensure_file_deleted(F3). ensure_file_deleted(F) -> case file:delete(F) of @@ -512,38 +508,35 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> File = - case has_deprecated_conf() of - true -> deprecated_override_conf_file(Opts); - false -> conf_file(Opts) + case has_deprecated_file() of + true -> deprecated_conf_file(Opts); + false -> cluster_hocon_file(Opts) end, load_hocon_file(File, map). -has_deprecated_conf() -> - DeprecatedFile = deprecated_cluster_conf_file(), +has_deprecated_file() -> + DeprecatedFile = deprecated_conf_file(#{override_to => cluster}), filelib:is_regular(DeprecatedFile). -deprecated_cluster_conf_file() -> - ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), - filename:join(filename:dirname(ClusterFile), "cluster-override.conf"). - -deprecated_override_conf_file(Opts) when is_map(Opts) -> +deprecated_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -deprecated_override_conf_file(Which) when is_atom(Which) -> +deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -conf_file(Opts) when is_map(Opts) -> +cluster_hocon_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of - local -> local_conf_file; - cluster -> cluster_conf_file + %% no local config file support + local -> undefined; + cluster -> cluster_hocon_file end, application:get_env(emqx, Key, undefined); -conf_file(Which) when is_atom(Which) -> +cluster_hocon_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. @@ -576,54 +569,17 @@ get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). -spec save_configs( - emqx_map_lib:config_key_path(), app_envs(), config(), raw_config(), raw_config(), update_opts() + app_envs(), config(), raw_config(), raw_config(), update_opts() ) -> ok. -save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> - Default = init_default(Paths0), - OverrideConf1 = remove_default_conf(OverrideConf, Default), +save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) -> %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - HasDeprecated = has_deprecated_conf(), - ok = save_to_override_conf(HasDeprecated, OverrideConf1, Opts), + HasDeprecatedFile = has_deprecated_file(), + ok = save_to_override_conf(HasDeprecatedFile, OverrideConf, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). -init_default(Paths0) -> - [Root | _] = [bin(Key) || Key <- Paths0], - SchemaMod = get_schema_mod(Root), - {_, {_, Schema}} = lists:keyfind(Root, 1, hocon_schema:roots(SchemaMod)), - fill_defaults(#{Root => schema_default(Schema)}). - -remove_default_conf(undefined, _) -> - undefined; -remove_default_conf(Conf, DefaultConf) when is_map(Conf) andalso is_map(DefaultConf) -> - maps:fold( - fun(Key, Value, Acc) -> - case maps:find(Key, DefaultConf) of - {ok, DefaultValue} -> - remove_default_conf(Value, DefaultValue, Key, Acc); - error -> - Acc - end - end, - Conf, - Conf - ). - -remove_default_conf(Value, Value, Key, Conf) -> - maps:remove(Key, Conf); -remove_default_conf(Value = #{}, DefaultValue = #{}, Key, Conf) -> - case remove_default_conf(Value, DefaultValue) of - SubValue when SubValue =:= #{} -> maps:remove(Key, Conf); - SubValue -> maps:put(Key, SubValue, Conf) - end; -remove_default_conf(Value, DefaultValue, Key, Conf) -> - case try_bin(DefaultValue) =:= try_bin(Value) of - true -> maps:remove(Key, Conf); - false -> Conf - end. - %% we ignore kernel app env, %% because the old app env will be used in emqx_config_logger:post_config_update/5 -define(IGNORE_APPS, [kernel]). @@ -643,7 +599,7 @@ save_to_override_conf(_, undefined, _) -> ok; %% TODO: Remove deprecated override conf file when 5.1 save_to_override_conf(true, RawConf, Opts) -> - case deprecated_override_conf_file(Opts) of + case deprecated_conf_file(Opts) of undefined -> ok; FileName -> @@ -661,7 +617,7 @@ save_to_override_conf(true, RawConf, Opts) -> end end; save_to_override_conf(false, RawConf, Opts) -> - case conf_file(Opts) of + case cluster_hocon_file(Opts) of undefined -> ok; FileName -> @@ -791,16 +747,6 @@ atom(Str) when is_list(Str) -> atom(Atom) when is_atom(Atom) -> Atom. -try_bin(Bin) when is_binary(Bin) -> Bin; -try_bin([Bin | _] = List) when is_binary(Bin) -> List; -try_bin([Atom | _] = List) when is_atom(Atom) -> [atom_to_binary(A) || A <- List]; -try_bin([Map | _] = Maps) when is_map(Map) -> Maps; -try_bin(Str) when is_list(Str) -> list_to_binary(Str); -try_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -try_bin(Int) when is_integer(Int) -> integer_to_binary(Int); -try_bin(Float) when is_float(Float) -> float_to_binary(Float); -try_bin(Term) -> Term. - bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). @@ -822,87 +768,3 @@ atom_conf_path(Path, ExpFun, OnFail) -> error(Err) end end. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -remove_default_conf_test() -> - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => 100}) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => <<"100">>}) - ), - ?assertEqual( - #{<<"def">> => 100}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => #{<<"bar">> => 100}}) - ), - ?assertEqual( - #{<<"def">> => #{<<"edf">> => 321}}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ - <<"def">> => #{<<"abc">> => 100, <<"edf">> => 123} - }) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ - <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => 321} - }) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => <<"true">>}}, #{ - <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => true} - }) - ), - ?assertEqual( - #{}, - remove_default_conf( - #{ - <<"bytes_in">> => - #{ - <<"capacity">> => infinity, - <<"initial">> => 0, - <<"rate">> => infinity - } - }, - #{ - <<"bytes_in">> => - #{ - <<"capacity">> => <<"infinity">>, - <<"initial">> => <<"0">>, - <<"rate">> => <<"infinity">> - } - } - ) - ), - ?assertEqual( - #{}, - remove_default_conf( - #{ - <<"limiter">> => #{ - <<"connection">> => - #{ - <<"capacity">> => 1000, - <<"initial">> => <<"0">>, - <<"rate">> => <<"1000/s">> - } - } - }, - #{ - <<"limiter">> => #{ - <<"connection">> => - #{ - <<"capacity">> => <<"1000">>, - <<"initial">> => <<"0">>, - <<"rate">> => <<"1000/s">> - } - } - } - ) - ), - ok. - --endif. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 3e18a172e..d5dc72acc 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -279,9 +279,7 @@ check_and_save_configs( OldConf = emqx_config:get_root(ConfKeyPath), case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of {ok, Result0} -> - ok = emqx_config:save_configs( - ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, Opts - ), + ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts), Result1 = return_change_result(ConfKeyPath, UpdateArgs), {ok, Result1#{post_config_update => Result0}}; Error -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index bc9da2a9e..6e56c85ed 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -314,8 +314,7 @@ stop_apps(Apps) -> ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), - application:unset_env(emqx, local_conf_file), - application:unset_env(emqx, cluster_conf_file), + application:unset_env(emqx, cluster_hocon_file), application:unset_env(gen_rpc, port_discovery), ok. @@ -467,7 +466,6 @@ force_set_config_file_paths(emqx, Paths) -> %% we need init cluster conf, so we can save the cluster conf to the file application:set_env(emqx, local_override_conf_file, "local_override.conf"), application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"), - application:set_env(emqx, local_conf_file, "local.hocon"), application:set_env(emqx, cluster_conf_file, "cluster.hocon"), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(_, _) -> diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index cea956825..deeee8d62 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -21,7 +21,6 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(LOCAL_CONF, "/tmp/local.conf"). -define(CLUSTER_CONF, "/tmp/cluster.conf"). -include_lib("eunit/include/eunit.hrl"). @@ -38,7 +37,6 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([]). init_per_testcase(_Case, Config) -> - _ = file:delete(?LOCAL_CONF), _ = file:delete(?CLUSTER_CONF), Config. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 45631619b..35a79ea6e 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,13 +60,13 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - HasDeprecateConf = emqx_config:has_deprecated_conf(), + HasDeprecateFile = emqx_config:has_deprecated_file(), #{ wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node, - has_deprecated_conf => HasDeprecateConf + has_deprecated_file => HasDeprecateFile } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of @@ -175,16 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, - HasDeprecatedConf = maps:get(has_deprecated_conf, Info, false), + HasDeprecatedFile = maps:get(has_deprecated_file, Info, false), ?SLOG(debug, #{ msg => "copy_cluster_conf_from_core_node_success", node => Node, - has_deprecated_conf => HasDeprecatedConf, + has_deprecated_file => HasDeprecatedFile, data_dir => emqx:data_dir(), tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( - HasDeprecatedConf, + HasDeprecatedFile, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a67d2a100..228462e73 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1029,8 +1029,7 @@ translation("emqx") -> {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, {"local_override_conf_file", fun tr_local_override_conf_file/1}, - {"cluster_conf_file", fun tr_cluster_conf_file/1}, - {"local_conf_file", fun tr_local_conf_file/1} + {"cluster_hocon_file", fun tr_cluster_hocon_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1081,14 +1080,11 @@ tr_cluster_override_conf_file(Conf) -> tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_conf_file(Conf, "local-overide.conf"). + tr_conf_file(Conf, "local-override.conf"). -tr_cluster_conf_file(Conf) -> +tr_cluster_hocon_file(Conf) -> tr_conf_file(Conf, "cluster.hocon"). -tr_local_conf_file(Conf) -> - tr_conf_file(Conf, "local.hocon"). - tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 21e22519e..825f01441 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -133,6 +133,7 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), + %% We set env both cluster.hocon and cluster-override.conf, but only one will be used application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. From bcce98990605eb33f93adca3600b8abd8025cbba Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Apr 2023 22:30:11 +0800 Subject: [PATCH 08/13] chore: update has_deprecated_file doc Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_config.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 75396d779..c73fdbc72 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -514,6 +514,8 @@ read_override_conf(#{} = Opts) -> end, load_hocon_file(File, map). +%% @doc Return `true' if this node is upgraded from older version which used cluster-override.conf for +%% cluster-wide config persistence. has_deprecated_file() -> DeprecatedFile = deprecated_conf_file(#{override_to => cluster}), filelib:is_regular(DeprecatedFile). From 7c5dead03a4a5b5074b16915469d90a96e424a0e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Apr 2023 22:30:35 +0800 Subject: [PATCH 09/13] chore: update cluster_hocon_file doc Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index c73fdbc72..b676b07e2 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -530,6 +530,7 @@ deprecated_conf_file(Opts) when is_map(Opts) -> deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). +%% The newer version cluster-wide config persistence file. cluster_hocon_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of From 8facd130f695464370949945a8a9cbb51d4624a4 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 22:45:28 +0800 Subject: [PATCH 10/13] chore: update cluster_hocon_file/0 function --- apps/emqx/src/emqx_config.erl | 22 +++++----------- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 26 +++++++------------ .../test/emqx_mgmt_api_configs_SUITE.erl | 17 +++++++++--- .../test/emqx_rule_funcs_SUITE.erl | 2 +- changes/ce/feat-10156.en.md | 7 +++++ 5 files changed, 37 insertions(+), 37 deletions(-) create mode 100644 changes/ce/feat-10156.en.md diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index b676b07e2..ab028cf7f 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -418,7 +418,7 @@ do_parse_hocon(false, Conf, IncDirs) -> true -> hocon:binary(Conf, Opts); false -> - ClusterFile = cluster_hocon_file(#{override_to => cluster}), + ClusterFile = cluster_hocon_file(), hocon:files([ClusterFile | Conf], Opts) end. @@ -493,7 +493,7 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> delete_override_conf_files() -> F1 = deprecated_conf_file(#{override_to => local}), F2 = deprecated_conf_file(#{override_to => cluster}), - F3 = cluster_hocon_file(#{override_to => cluster}), + F3 = cluster_hocon_file(), ok = ensure_file_deleted(F1), ok = ensure_file_deleted(F2), ok = ensure_file_deleted(F3). @@ -510,7 +510,7 @@ read_override_conf(#{} = Opts) -> File = case has_deprecated_file() of true -> deprecated_conf_file(Opts); - false -> cluster_hocon_file(Opts) + false -> cluster_hocon_file() end, load_hocon_file(File, map). @@ -531,16 +531,8 @@ deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). %% The newer version cluster-wide config persistence file. -cluster_hocon_file(Opts) when is_map(Opts) -> - Key = - case maps:get(override_to, Opts, cluster) of - %% no local config file support - local -> undefined; - cluster -> cluster_hocon_file - end, - application:get_env(emqx, Key, undefined); -cluster_hocon_file(Which) when is_atom(Which) -> - application:get_env(emqx, Which, undefined). +cluster_hocon_file() -> + application:get_env(emqx, cluster_hocon_file, undefined). -spec save_schema_mod_and_names(module()) -> ok. save_schema_mod_and_names(SchemaMod) -> @@ -619,8 +611,8 @@ save_to_override_conf(true, RawConf, Opts) -> {error, Reason} end end; -save_to_override_conf(false, RawConf, Opts) -> - case cluster_hocon_file(Opts) of +save_to_override_conf(false, RawConf, _Opts) -> + case cluster_hocon_file() of undefined -> ok; FileName -> diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 825f01441..583405158 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -59,14 +59,15 @@ t_copy_new_data_dir(_Config) -> %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), try + File = "/configs/cluster.hocon", assert_config_load_done(Nodes), - rpc:call(First, ?MODULE, create_data_dir, []), + rpc:call(First, ?MODULE, create_data_dir, [File]), {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes, "/configs/cluster.hocon"), + assert_data_copy_done(Nodes, File), stop_cluster(Nodes), ok after @@ -82,14 +83,15 @@ t_copy_deprecated_data_dir(_Config) -> %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), try + File = "/configs/cluster-override.conf", assert_config_load_done(Nodes), - rpc:call(First, ?MODULE, create_deprecated_data_dir, []), + rpc:call(First, ?MODULE, create_data_dir, [File]), {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes, "/configs/cluster-override.conf"), + assert_data_copy_done(Nodes, File), stop_cluster(Nodes), ok after @@ -100,7 +102,7 @@ t_copy_deprecated_data_dir(_Config) -> %% Helper functions %%------------------------------------------------------------------------------ -create_data_dir() -> +create_data_dir(File) -> Node = atom_to_list(node()), ok = filelib:ensure_dir(Node ++ "/certs/"), ok = filelib:ensure_dir(Node ++ "/authz/"), @@ -108,17 +110,7 @@ create_data_dir() -> ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). - -create_deprecated_data_dir() -> - Node = atom_to_list(node()), - ok = filelib:ensure_dir(Node ++ "/certs/"), - ok = filelib:ensure_dir(Node ++ "/authz/"), - ok = filelib:ensure_dir(Node ++ "/configs/"), - ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), - ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), - Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + ok = file:write_file(Node ++ File, Telemetry). set_data_dir_env() -> Node = atom_to_list(node()), @@ -134,7 +126,7 @@ set_data_dir_env() -> application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), %% We set env both cluster.hocon and cluster-override.conf, but only one will be used - application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_hocon_file, Node ++ "/configs/cluster.hocon"), application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index f5ee4e5be..fc1620fec 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -62,8 +62,8 @@ t_update(_Config) -> ?assertEqual(BusyPort, not BusyPort1), assert_busy_port(BusyPort1), %% Make sure the override config is updated, and remove the default value. - ?assertEqual( - #{<<"vm">> => #{<<"busy_port">> => BusyPort1}}, + ?assertMatch( + #{<<"vm">> := #{<<"busy_port">> := BusyPort1}}, maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster})) ), @@ -136,7 +136,7 @@ t_global_zone(_Config) -> {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), %% Make sure the override config is updated, and remove the default value. - ?assertEqual(#{<<"max_qos_allowed">> => 1}, read_conf(<<"mqtt">>)), + ?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)), BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), @@ -155,7 +155,16 @@ t_global_zone(_Config) -> DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), {ok, #{}} = update_global_zone(DefaultZones), - ?assertEqual(undefined, read_conf(<<"mqtt">>)), + #{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}), + Default = maps:map( + fun + (_, V) when is_boolean(V) -> V; + (_, V) when is_atom(V) -> atom_to_binary(V); + (_, V) -> V + end, + Mqtt + ), + ?assertEqual(Default, read_conf(<<"mqtt">>)), ok. get_global_zone() -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 94adb3506..ee798868e 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -686,7 +686,7 @@ t_jq(_) -> %% Got timeout as expected got_timeout end, - ConfigRootKey = emqx_rule_engine_schema:namespace(), + _ConfigRootKey = emqx_rule_engine_schema:namespace(), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md new file mode 100644 index 000000000..bc864fcc3 --- /dev/null +++ b/changes/ce/feat-10156.en.md @@ -0,0 +1,7 @@ +Change the priority of the configuration: +1. If it is a new installation of EMQX, the priority of s configuration is `ENV > emqx.conf > HTTP API`. +2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. + +Deprecated data/configs/local-override.conf. + +Stabilizing the HTTP API for hot updates. From 8461551f51cf5ee56ac315172f06bf8b75306bdb Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sat, 15 Apr 2023 06:56:33 +0800 Subject: [PATCH 11/13] test: rename module from emqx_map_lib to emqx_utils_maps --- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 5fe9f881e..db735bb27 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -159,7 +159,7 @@ t_global_zone(_Config) -> ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), - DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), + DefaultZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), {ok, #{}} = update_global_zone(DefaultZones), #{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}), Default = maps:map( @@ -324,7 +324,7 @@ reset_config(Name, Key) -> read_conf(RootKeys) when is_list(RootKeys) -> case emqx_config:read_override_conf(#{override_to => cluster}) of undefined -> undefined; - Conf -> emqx_map_lib:deep_get(RootKeys, Conf, undefined) + Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined) end; read_conf(RootKey) -> read_conf([RootKey]). From e1faae451a27801fb4ae23cc916f2ba9774aab20 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 15 Apr 2023 18:17:28 +0800 Subject: [PATCH 12/13] chore: Update changes/ce/feat-10156.en.md Co-authored-by: JianBo He --- changes/ce/feat-10156.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md index bc864fcc3..c1291a7ce 100644 --- a/changes/ce/feat-10156.en.md +++ b/changes/ce/feat-10156.en.md @@ -1,5 +1,5 @@ Change the priority of the configuration: -1. If it is a new installation of EMQX, the priority of s configuration is `ENV > emqx.conf > HTTP API`. +1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. 2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. Deprecated data/configs/local-override.conf. From 454d244535aa7bee1ff2630c43e20874e5c64558 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 16 Apr 2023 09:58:22 +0800 Subject: [PATCH 13/13] chore: update changes/ce/feat-10156.en.md Co-authored-by: JianBo He --- changes/ce/feat-10156.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md index c1291a7ce..589578f26 100644 --- a/changes/ce/feat-10156.en.md +++ b/changes/ce/feat-10156.en.md @@ -1,6 +1,6 @@ Change the priority of the configuration: 1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. -2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. +2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`. Deprecated data/configs/local-override.conf.