From 5f4ea3b6d8621ac667b7af121fac452617cea6c7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 7 Apr 2023 09:17:54 +0800 Subject: [PATCH] feat: deprecated cluster-override.conf --- apps/emqx/src/emqx_config.erl | 142 ++++++++++++++++-------- apps/emqx_conf/src/emqx_conf_app.erl | 29 +++-- apps/emqx_conf/src/emqx_conf_schema.erl | 16 ++- 3 files changed, 127 insertions(+), 60 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 41e38f184..68a8563d8 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,6 +24,7 @@ init_load/2, init_load/3, read_override_conf/1, + has_deprecated_conf/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, @@ -32,7 +33,7 @@ save_configs/6, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/2 + save_to_override_conf/3 ]). -export([raw_conf_with_default/4]). -export([remove_default_conf/2]). @@ -331,14 +332,35 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(SchemaMod, parse_hocon(Conf), Opts); init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - RootNames = get_root_names(), - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll). + case has_deprecated_conf() of + false -> + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); + true -> + %% deprecated conf will be removed in 5.1 + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + Overrides = read_override_confs(), + RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), + RootNames = get_root_names(), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll) + end. + +%% @doc Read merged cluster + local overrides. +read_override_confs() -> + ClusterOverrides = read_override_conf(#{override_to => cluster}), + LocalOverrides = read_override_conf(#{override_to => local}), + hocon:deep_merge(ClusterOverrides, LocalOverrides). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> @@ -369,11 +391,9 @@ schema_default(Schema) -> end. parse_hocon(Conf) -> - %% merge cluster-override.conf to local-override.conf - %% cluster-override.conf is deprecated, now is cluster.conf - merge_deprecated_cluster_override_to_local_override(), + HasDeprecatedConf = has_deprecated_conf(), IncDirs = include_dirs(), - case do_parse_hocon(Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedConf, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> @@ -387,14 +407,20 @@ parse_hocon(Conf) -> error(failed_to_load_hocon_conf) end. -do_parse_hocon(Conf, IncDirs) -> +do_parse_hocon(true, Conf, IncDirs) -> + Opts = #{format => map, include_dirs => IncDirs}, + case is_binary(Conf) of + true -> hocon:binary(Conf, Opts); + false -> hocon:files(Conf, Opts) + end; +do_parse_hocon(false, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of true -> hocon:binary(Conf, Opts); false -> - LocalFile = override_conf_file(#{override_to => local}), - ClusterFile = override_conf_file(#{override_to => cluster}), + LocalFile = deprecated_override_conf_file(#{override_to => local}), + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) end. @@ -467,10 +493,14 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = override_conf_file(#{override_to => local}), - F2 = override_conf_file(#{override_to => cluster}), + F1 = deprecated_override_conf_file(#{override_to => local}), + F2 = deprecated_override_conf_file(#{override_to => cluster}), + F3 = conf_file(#{override_to => local}), + F4 = conf_file(#{override_to => cluster}), ok = ensure_file_deleted(F1), - ok = ensure_file_deleted(F2). + ok = ensure_file_deleted(F2), + ok = ensure_file_deleted(F3), + ok = ensure_file_deleted(F4). ensure_file_deleted(F) -> case file:delete(F) of @@ -481,24 +511,39 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> - File = override_conf_file(Opts), + File = + case has_deprecated_conf() of + true -> deprecated_override_conf_file(Opts); + false -> conf_file(Opts) + end, load_hocon_file(File, map). -read_deprecated_override_conf() -> - ClusterFile = override_conf_file(#{override_to => cluster}), - DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"), - Conf = load_hocon_file(DeprecatedFile, map), - _ = file:delete(DeprecatedFile), - Conf. +has_deprecated_conf() -> + DeprecatedFile = deprecated_cluster_conf_file(), + filelib:is_regular(DeprecatedFile). -override_conf_file(Opts) when is_map(Opts) -> +deprecated_cluster_conf_file() -> + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), + filename:join(filename:dirname(ClusterFile), "cluster-override.conf"). + +deprecated_override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -override_conf_file(Which) when is_atom(Which) -> +deprecated_override_conf_file(Which) when is_atom(Which) -> + application:get_env(emqx, Which, undefined). + +conf_file(Opts) when is_map(Opts) -> + Key = + case maps:get(override_to, Opts, cluster) of + local -> local_conf_file; + cluster -> cluster_conf_file + end, + application:get_env(emqx, Key, undefined); +conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. @@ -539,7 +584,8 @@ save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> OverrideConf1 = remove_default_conf(OverrideConf, Default), %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf1, Opts), + HasDeprecated = has_deprecated_conf(), + ok = save_to_override_conf(HasDeprecated, OverrideConf1, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). @@ -592,11 +638,12 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put(Conf), ?MODULE:put_raw(RawConf). --spec save_to_override_conf(raw_config(), update_opts()) -> ok | {error, term()}. -save_to_override_conf(undefined, _) -> +-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}. +save_to_override_conf(_, undefined, _) -> ok; -save_to_override_conf(RawConf, Opts) -> - case override_conf_file(Opts) of +%% TODO: Remove deprecated override conf file when 5.1 +save_to_override_conf(true, RawConf, Opts) -> + case deprecated_override_conf_file(Opts) of undefined -> ok; FileName -> @@ -612,6 +659,24 @@ save_to_override_conf(RawConf, Opts) -> }), {error, Reason} end + end; +save_to_override_conf(false, RawConf, Opts) -> + case conf_file(Opts) of + undefined -> + ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_save_conf_file", + filename => FileName, + reason => Reason + }), + {error, Reason} + end end. add_handlers() -> @@ -758,17 +823,6 @@ atom_conf_path(Path, ExpFun, OnFail) -> end end. -merge_deprecated_cluster_override_to_local_override() -> - case read_deprecated_override_conf() of - DeprecatedConf when DeprecatedConf =/= #{} -> - LocalOverrides = read_override_conf(#{override_to => local}), - MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf), - _ = save_to_override_conf(MergedConf, #{override_to => local}), - ok; - _ -> - ok - end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 51fc5c2e2..45631619b 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,7 +60,14 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - #{wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node} + HasDeprecateConf = emqx_config:has_deprecated_conf(), + #{ + wall_clock => WallClock, + conf => Conf, + tnx_id => TnxId, + node => Node, + has_deprecated_conf => HasDeprecateConf + } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of {atomic, Res} -> {ok, Res}; @@ -153,10 +160,10 @@ copy_override_conf_from_core_node() -> {ok, ?DEFAULT_INIT_TXN_ID}; false -> %% retry in some time - Jitter = rand:uniform(2_000), - Timeout = 10_000 + Jitter, + Jitter = rand:uniform(2000), + Timeout = 10000 + Jitter, ?SLOG(info, #{ - msg => "copy_override_conf_from_core_node_retry", + msg => "copy_cluster_conf_from_core_node_retry", timeout => Timeout, nodes => Nodes, failed => Failed, @@ -168,18 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, + HasDeprecatedConf = maps:get(has_deprecated_conf, Info, false), ?SLOG(debug, #{ - msg => "copy_override_conf_from_core_node_success", + msg => "copy_cluster_conf_from_core_node_success", node => Node, - cluster_override_conf_file => application:get_env( - emqx, cluster_override_conf_file - ), - local_override_conf_file => application:get_env( - emqx, local_override_conf_file - ), - data_dir => emqx:data_dir() + has_deprecated_conf => HasDeprecatedConf, + data_dir => emqx:data_dir(), + tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( + HasDeprecatedConf, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ed246b4ae..a67d2a100 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1028,7 +1028,9 @@ translation("emqx") -> [ {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, - {"local_override_conf_file", fun tr_local_override_conf_file/1} + {"local_override_conf_file", fun tr_local_override_conf_file/1}, + {"cluster_conf_file", fun tr_cluster_conf_file/1}, + {"local_conf_file", fun tr_local_conf_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1076,12 +1078,18 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster.conf"). + tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local.conf"). + tr_conf_file(Conf, "local-overide.conf"). -tr_override_conf_file(Conf, Filename) -> +tr_cluster_conf_file(Conf) -> + tr_conf_file(Conf, "cluster.hocon"). + +tr_local_conf_file(Conf) -> + tr_conf_file(Conf, "local.hocon"). + +tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable [_ | _] = DataDir,