yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10156 from zhongwencool/conf-refactor 

feat: configuration priority ENV > emqx.conf > API
This commit is contained in:
zhongwencool 2023-04-16 13:45:35 +08:00 committed by GitHub
parent bcbeae1da5 454d244535
commit 69d1a35c90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 297 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/rebar.config
View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

23
apps/emqx/src/config/emqx_config_logger.erl
View File

@ -32,25 +32,15 @@ remove_handler() ->
ok = emqx_config_handler:remove_handler(?LOG),
ok.
%% refresh logger config when booting, the override config may have changed after node start.
%% refresh logger config when booting, the cluster config may have changed after node start.
%% Kernel's app env is confirmed before the node starts,
%% but we only copy cluster-override.conf from other node after this node starts,
%% but we only copy cluster.conf from other node after this node starts,
%% so we need to refresh the logger config after this node starts.
%% It will not affect the logger config when cluster-override.conf is unchanged.
%% It will not affect the logger config when cluster.conf is unchanged.
refresh_config() ->
Overrides = emqx_config:read_override_confs(),
refresh_config(Overrides).
refresh_config(#{<<"log">> := _}) ->
%% read the checked config
LogConfig = emqx:get_config(?LOG, undefined),
Conf = #{log => LogConfig},
ok = do_refresh_config(Conf);
refresh_config(_) ->
%% No config override found for 'log', do nothing
%% because the 'kernel' app should already be configured
%% from the base configs. i.e. emqx.conf + env vars
ok.
do_refresh_config(#{log => LogConfig}).
%% this call is shared between initial config refresh at boot
%% and dynamic config update from HTTP API
@ -61,10 +51,9 @@ do_refresh_config(Conf) ->
ok = maybe_update_log_level(Level),
ok.
%% always refresh config when the override config is changed
post_config_update(?LOG, _Req, NewConf, _OldConf, _AppEnvs) ->
ok = do_refresh_config(#{log => NewConf});
post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
ok.
do_refresh_config(#{log => NewConf}).
maybe_update_log_level(NewLevel) ->
OldLevel = emqx_logger:get_primary_log_level(),

110
apps/emqx/src/emqx_config.erl
View File

@ -24,7 +24,7 @@
init_load/2,
init_load/3,
read_override_conf/1,
read_override_confs/0,
has_deprecated_file/0,
delete_override_conf_files/0,
check_config/2,
fill_defaults/1,
@ -33,8 +33,10 @@
save_configs/5,
save_to_app_env/1,
save_to_config_map/2,
save_to_override_conf/2
save_to_override_conf/3
]).
-export([raw_conf_with_default/4]).
-export([merge_envs/2]).
-export([
get_root/1,
@ -326,9 +328,12 @@ init_load(SchemaMod, ConfFiles) ->
%% in the rear of the list overrides prior values.
-spec init_load(module(), [string()] | binary() | hocon:config()) -> ok.
init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) ->
init_load(SchemaMod, parse_hocon(Conf), Opts);
init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) ->
ok = save_schema_mod_and_names(SchemaMod),
HasDeprecatedFile = has_deprecated_file(),
RawConf = parse_hocon(HasDeprecatedFile, Conf),
init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts).
init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) ->
%% deprecated conf will be removed in 5.1
%% Merge environment variable overrides on top
RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
Overrides = read_override_confs(),
@ -338,6 +343,16 @@ init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) ->
%% check configs against the schema
{AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}),
save_to_app_env(AppEnvs),
ok = save_to_config_map(CheckedConf, RawConfAll);
init_load(false, SchemaMod, RawConf, Opts) when is_map(RawConf) ->
ok = save_schema_mod_and_names(SchemaMod),
RootNames = get_root_names(),
%% Merge environment variable overrides on top
RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts),
%% check configs against the schema
{AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}),
save_to_app_env(AppEnvs),
ok = save_to_config_map(CheckedConf, RawConfAll).
%% @doc Read merged cluster + local overrides.
@ -374,27 +389,37 @@ schema_default(Schema) ->
#{}
end.
parse_hocon(Conf) ->
parse_hocon(HasDeprecatedFile, Conf) ->
IncDirs = include_dirs(),
case do_parse_hocon(Conf, IncDirs) of
case do_parse_hocon(HasDeprecatedFile, Conf, IncDirs) of
{ok, HoconMap} ->
HoconMap;
{error, Reason} ->
?SLOG(error, #{
msg => "failed_to_load_hocon_conf",
msg => "failed_to_load_hocon_file",
reason => Reason,
pwd => file:get_cwd(),
include_dirs => IncDirs,
config_file => Conf
}),
error(failed_to_load_hocon_conf)
error(failed_to_load_hocon_file)
end.
do_parse_hocon(Conf, IncDirs) ->
do_parse_hocon(true, Conf, IncDirs) ->
Opts = #{format => map, include_dirs => IncDirs},
case is_binary(Conf) of
true -> hocon:binary(Conf, Opts);
false -> hocon:files(Conf, Opts)
end;
do_parse_hocon(false, Conf, IncDirs) ->
Opts = #{format => map, include_dirs => IncDirs},
case is_binary(Conf) of
%% only use in test
true ->
hocon:binary(Conf, Opts);
false ->
ClusterFile = cluster_hocon_file(),
hocon:files([ClusterFile | Conf], Opts)
end.
include_dirs() ->
@ -466,10 +491,12 @@ fill_defaults(SchemaMod, RawConf, Opts0) ->
%% Delete override config files.
-spec delete_override_conf_files() -> ok.
delete_override_conf_files() ->
F1 = override_conf_file(#{override_to => local}),
F2 = override_conf_file(#{override_to => cluster}),
F1 = deprecated_conf_file(#{override_to => local}),
F2 = deprecated_conf_file(#{override_to => cluster}),
F3 = cluster_hocon_file(),
ok = ensure_file_deleted(F1),
ok = ensure_file_deleted(F2).
ok = ensure_file_deleted(F2),
ok = ensure_file_deleted(F3).
ensure_file_deleted(F) ->
case file:delete(F) of
@ -480,19 +507,33 @@ ensure_file_deleted(F) ->
-spec read_override_conf(map()) -> raw_config().
read_override_conf(#{} = Opts) ->
File = override_conf_file(Opts),
File =
case has_deprecated_file() of
true -> deprecated_conf_file(Opts);
false -> cluster_hocon_file()
end,
load_hocon_file(File, map).
override_conf_file(Opts) when is_map(Opts) ->
%% @doc Return `true' if this node is upgraded from older version which used cluster-override.conf for
%% cluster-wide config persistence.
has_deprecated_file() ->
DeprecatedFile = deprecated_conf_file(#{override_to => cluster}),
filelib:is_regular(DeprecatedFile).
deprecated_conf_file(Opts) when is_map(Opts) ->
Key =
case maps:get(override_to, Opts, cluster) of
local -> local_override_conf_file;
cluster -> cluster_override_conf_file
end,
application:get_env(emqx, Key, undefined);
override_conf_file(Which) when is_atom(Which) ->
deprecated_conf_file(Which) when is_atom(Which) ->
application:get_env(emqx, Which, undefined).
%% The newer version cluster-wide config persistence file.
cluster_hocon_file() ->
application:get_env(emqx, cluster_hocon_file, undefined).
-spec save_schema_mod_and_names(module()) -> ok.
save_schema_mod_and_names(SchemaMod) ->
RootNames = hocon_schema:root_names(SchemaMod),
@ -522,11 +563,15 @@ get_schema_mod(RootName) ->
get_root_names() ->
maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})).
-spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok.
-spec save_configs(
app_envs(), config(), raw_config(), raw_config(), update_opts()
) -> ok.
save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) ->
%% We first try to save to override.conf, because saving to files is more error prone
%% We first try to save to files, because saving to files is more error prone
%% than saving into memory.
ok = save_to_override_conf(OverrideConf, Opts),
HasDeprecatedFile = has_deprecated_file(),
ok = save_to_override_conf(HasDeprecatedFile, OverrideConf, Opts),
save_to_app_env(AppEnvs),
save_to_config_map(Conf, RawConf).
@ -544,11 +589,12 @@ save_to_config_map(Conf, RawConf) ->
?MODULE:put(Conf),
?MODULE:put_raw(RawConf).
-spec save_to_override_conf(raw_config(), update_opts()) -> ok | {error, term()}.
save_to_override_conf(undefined, _) ->
-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
save_to_override_conf(_, undefined, _) ->
ok;
save_to_override_conf(RawConf, Opts) ->
case override_conf_file(Opts) of
%% TODO: Remove deprecated override conf file when 5.1
save_to_override_conf(true, RawConf, Opts) ->
case deprecated_conf_file(Opts) of
undefined ->
ok;
FileName ->
@ -564,6 +610,24 @@ save_to_override_conf(RawConf, Opts) ->
}),
{error, Reason}
end
end;
save_to_override_conf(false, RawConf, _Opts) ->
case cluster_hocon_file() of
undefined ->
ok;
FileName ->
ok = filelib:ensure_dir(FileName),
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
ok ->
ok;
{error, Reason} ->
?SLOG(error, #{
msg => "failed_to_save_conf_file",
filename => FileName,
reason => Reason
}),
{error, Reason}
end
end.
add_handlers() ->

123
apps/emqx/src/emqx_config_handler.erl
View File

@ -43,7 +43,6 @@
terminate/2,
code_change/3
]).
-export([is_mutable/3]).
-define(MOD, {mod}).
-define(WKEY, '?').
@ -230,26 +229,15 @@ process_update_request([_], _Handlers, {remove, _Opts}) ->
process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
BinKeyPath = bin_path(ConfKeyPath),
case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of
allow ->
NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf),
OverrideConf = remove_from_override_config(BinKeyPath, Opts),
{ok, NewRawConf, OverrideConf, Opts};
{deny, Reason} ->
{error, {permission_denied, Reason}}
end;
NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf),
OverrideConf = remove_from_override_config(BinKeyPath, Opts),
{ok, NewRawConf, OverrideConf, Opts};
process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
{ok, NewRawConf} ->
BinKeyPath = bin_path(ConfKeyPath),
case check_permissions(update, BinKeyPath, NewRawConf, Opts) of
allow ->
OverrideConf = merge_to_override_config(NewRawConf, Opts),
{ok, NewRawConf, OverrideConf, Opts};
{deny, Reason} ->
{error, {permission_denied, Reason}}
end;
OverrideConf = merge_to_override_config(NewRawConf, Opts),
{ok, NewRawConf, OverrideConf, Opts};
Error ->
Error
end.
@ -271,8 +259,10 @@ do_update_config(
SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf),
SubHandlers = get_sub_handlers(ConfKey, Handlers),
case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of
{ok, NewUpdateReq} -> merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
Error -> Error
{ok, NewUpdateReq} ->
merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
Error ->
Error
end.
check_and_save_configs(
@ -546,98 +536,3 @@ load_prev_handlers() ->
save_handlers(Handlers) ->
application:set_env(emqx, ?MODULE, Handlers).
check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) ->
allow;
check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) ->
case emqx_utils_maps:deep_find(ConfKeyPath, NewRawConf) of
{ok, NewRaw} ->
LocalOverride = emqx_config:read_override_conf(#{override_to => local}),
case emqx_utils_maps:deep_find(ConfKeyPath, LocalOverride) of
{ok, LocalRaw} ->
case is_mutable(Action, NewRaw, LocalRaw) of
ok ->
allow;
{error, Error} ->
?SLOG(error, #{
msg => "prevent_remove_local_override_conf",
config_key_path => ConfKeyPath,
error => Error
}),
{deny, "Disable changed from local-override.conf"}
end;
{not_found, _, _} ->
allow
end;
{not_found, _, _} ->
allow
end.
is_mutable(Action, NewRaw, LocalRaw) ->
try
KeyPath = [],
is_mutable(KeyPath, Action, NewRaw, LocalRaw)
catch
throw:Error -> Error
end.
-define(REMOVE_FAILED, "remove_failed").
-define(UPDATE_FAILED, "update_failed").
is_mutable(KeyPath, Action, New = #{}, Local = #{}) ->
maps:foreach(
fun(Key, SubLocal) ->
case maps:find(Key, New) of
error -> ok;
{ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal)
end
end,
Local
);
is_mutable(KeyPath, remove, Update, Origin) ->
throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}});
is_mutable(_KeyPath, update, Val, Val) ->
ok;
is_mutable(KeyPath, update, Update, Origin) ->
throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
is_mutable_update_test() ->
Action = update,
?assertEqual(ok, is_mutable(Action, #{}, #{})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})),
?assertEqual(
{error, {?UPDATE_FAILED, [a, b, c], 1, 2}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
),
?assertEqual(
{error, {?UPDATE_FAILED, [a, b, d], 2, 3}},
is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
),
ok.
is_mutable_remove_test() ->
Action = remove,
?assertEqual(ok, is_mutable(Action, #{}, #{})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})
),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 2}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
),
ok.
-endif.

8
apps/emqx/test/emqx_authentication_SUITE.erl
View File

@ -95,13 +95,17 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
LogLevel = emqx_logger:get_primary_log_level(),
ok = emqx_logger:set_log_level(debug),
application:set_env(ekka, strict_mode, true),
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
Config.
[{log_level, LogLevel} | Config].
end_per_suite(_) ->
end_per_suite(Config) ->
emqx_common_test_helpers:stop_apps([]),
LogLevel = ?config(log_level),
emqx_logger:set_log_level(LogLevel),
ok.
init_per_testcase(Case, Config) ->

5
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -314,6 +314,7 @@ stop_apps(Apps) ->
ok = emqx_config:delete_override_conf_files(),
application:unset_env(emqx, local_override_conf_file),
application:unset_env(emqx, cluster_override_conf_file),
application:unset_env(emqx, cluster_hocon_file),
application:unset_env(gen_rpc, port_discovery),
ok.
@ -462,6 +463,10 @@ force_set_config_file_paths(emqx_conf, [Path] = Paths) ->
ok = file:write_file(Path, Bin, [append]),
application:set_env(emqx, config_files, Paths);
force_set_config_file_paths(emqx, Paths) ->
%% we need init cluster conf, so we can save the cluster conf to the file
application:set_env(emqx, local_override_conf_file, "local_override.conf"),
application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"),
application:set_env(emqx, cluster_conf_file, "cluster.hocon"),
application:set_env(emqx, config_files, Paths);
force_set_config_file_paths(_, _) ->
ok.

64
apps/emqx/test/emqx_config_handler_SUITE.erl
View File

@ -21,8 +21,7 @@
-define(MOD, {mod}).
-define(WKEY, '?').
-define(LOCAL_CONF, "/tmp/local-override.conf").
-define(CLUSTER_CONF, "/tmp/cluster-override.conf").
-define(CLUSTER_CONF, "/tmp/cluster.conf").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -38,7 +37,6 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(_Case, Config) ->
_ = file:delete(?LOCAL_CONF),
_ = file:delete(?CLUSTER_CONF),
Config.
@ -200,62 +198,6 @@ t_sub_key_update_remove(_Config) ->
ok = emqx_config_handler:remove_handler(KeyPath2),
ok.
t_local_override_update_remove(_Config) ->
application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF),
application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF),
KeyPath = [sysmon, os, cpu_high_watermark],
ok = emqx_config_handler:add_handler(KeyPath, ?MODULE),
LocalOpts = #{override_to => local},
{ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts),
?assertMatch(
#{
config := 0.7,
post_config_update := #{},
raw_config := <<"70%">>
},
Res
),
ClusterOpts = #{override_to => cluster},
?assertMatch(
{error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts)
),
?assertMatch(0.7, emqx:get_config(KeyPath)),
KeyPath2 = [sysmon, os, cpu_low_watermark],
ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE),
?assertMatch(
{error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts)
),
%% remove
?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)),
?assertEqual(
{ok, #{post_config_update => #{}}},
emqx:remove_config(KeyPath, #{override_to => local})
),
?assertEqual(
{ok, #{post_config_update => #{}}},
emqx:remove_config(KeyPath)
),
?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)),
OSKey = maps:keys(emqx:get_raw_config([sysmon, os])),
?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)),
?assert(length(OSKey) > 0),
?assertEqual(
{ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}},
emqx:reset_config(KeyPath, ClusterOpts)
),
OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])),
?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)),
?assert(length(OSKey1) > 1),
ok = emqx_config_handler:remove_handler(KeyPath),
ok = emqx_config_handler:remove_handler(KeyPath2),
application:unset_env(emqx, local_override_conf_file),
application:unset_env(emqx, cluster_override_conf_file),
ok.
t_check_failed(_Config) ->
KeyPath = [sysmon, os, cpu_check_interval],
Opts = #{rawconf_with_defaults => true},
@ -426,9 +368,9 @@ wait_for_new_pid() ->
callback_error(FailedPath, Update, Error) ->
Opts = #{rawconf_with_defaults => true},
ok = emqx_config_handler:add_handler(FailedPath, ?MODULE),
Old = emqx:get_raw_config(FailedPath),
Old = emqx:get_raw_config(FailedPath, undefined),
?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)),
New = emqx:get_raw_config(FailedPath),
New = emqx:get_raw_config(FailedPath, undefined),
?assertEqual(Old, New),
ok = emqx_config_handler:remove_handler(FailedPath),
ok.

29
apps/emqx_conf/src/emqx_conf_app.erl
View File

@ -60,7 +60,14 @@ get_override_config_file() ->
TnxId = emqx_cluster_rpc:get_node_tnx_id(Node),
WallClock = erlang:statistics(wall_clock),
Conf = emqx_config_handler:get_raw_cluster_override_conf(),
#{wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node}
HasDeprecateFile = emqx_config:has_deprecated_file(),
#{
wall_clock => WallClock,
conf => Conf,
tnx_id => TnxId,
node => Node,
has_deprecated_file => HasDeprecateFile
}
end,
case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of
{atomic, Res} -> {ok, Res};
@ -153,10 +160,10 @@ copy_override_conf_from_core_node() ->
{ok, ?DEFAULT_INIT_TXN_ID};
false ->
%% retry in some time
Jitter = rand:uniform(2_000),
Timeout = 10_000 + Jitter,
Jitter = rand:uniform(2000),
Timeout = 10000 + Jitter,
?SLOG(info, #{
msg => "copy_override_conf_from_core_node_retry",
msg => "copy_cluster_conf_from_core_node_retry",
timeout => Timeout,
nodes => Nodes,
failed => Failed,
@ -168,18 +175,16 @@ copy_override_conf_from_core_node() ->
_ ->
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
HasDeprecatedFile = maps:get(has_deprecated_file, Info, false),
?SLOG(debug, #{
msg => "copy_override_conf_from_core_node_success",
msg => "copy_cluster_conf_from_core_node_success",
node => Node,
cluster_override_conf_file => application:get_env(
emqx, cluster_override_conf_file
),
local_override_conf_file => application:get_env(
emqx, local_override_conf_file
),
data_dir => emqx:data_dir()
has_deprecated_file => HasDeprecatedFile,
data_dir => emqx:data_dir(),
tnx_id => TnxId
}),
ok = emqx_config:save_to_override_conf(
HasDeprecatedFile,
RawOverrideConf,
#{override_to => cluster}
),

12
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -1029,7 +1029,8 @@ translation("emqx") ->
[
{"config_files", fun tr_config_files/1},
{"cluster_override_conf_file", fun tr_cluster_override_conf_file/1},
{"local_override_conf_file", fun tr_local_override_conf_file/1}
{"local_override_conf_file", fun tr_local_override_conf_file/1},
{"cluster_hocon_file", fun tr_cluster_hocon_file/1}
];
translation("gen_rpc") ->
[{"default_client_driver", fun tr_default_config_driver/1}];
@ -1077,12 +1078,15 @@ tr_config_files(_Conf) ->
end.
tr_cluster_override_conf_file(Conf) ->
tr_override_conf_file(Conf, "cluster-override.conf").
tr_conf_file(Conf, "cluster-override.conf").
tr_local_override_conf_file(Conf) ->
tr_override_conf_file(Conf, "local-override.conf").
tr_conf_file(Conf, "local-override.conf").
tr_override_conf_file(Conf, Filename) ->
tr_cluster_hocon_file(Conf) ->
tr_conf_file(Conf, "cluster.hocon").
tr_conf_file(Conf, Filename) ->
DataDir = conf_get("node.data_dir", Conf),
%% assert, this config is not nullable
[_ | _] = DataDir,

55
apps/emqx_conf/test/emqx_conf_app_SUITE.erl
View File

@ -27,7 +27,7 @@ all() ->
t_copy_conf_override_on_restarts(_Config) ->
ct:timetrap({seconds, 120}),
snabbkaffe:fix_ct_logging(),
Cluster = cluster([core, core, core]),
Cluster = cluster([cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})]),
%% 1. Start all nodes
Nodes = start_cluster(Cluster),
@ -41,7 +41,7 @@ t_copy_conf_override_on_restarts(_Config) ->
%% crash and eventually all nodes should be ready.
start_cluster_async(Cluster),
timer:sleep(15_000),
timer:sleep(15000),
assert_config_load_done(Nodes),
@ -50,23 +50,48 @@ t_copy_conf_override_on_restarts(_Config) ->
stop_cluster(Nodes)
end.
t_copy_data_dir(_Config) ->
t_copy_new_data_dir(_Config) ->
net_kernel:start(['master1@127.0.0.1', longnames]),
ct:timetrap({seconds, 120}),
snabbkaffe:fix_ct_logging(),
Cluster = cluster([{core, copy1}, {core, copy2}, {core, copy3}]),
Cluster = cluster([cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})]),
%% 1. Start all nodes
[First | Rest] = Nodes = start_cluster(Cluster),
try
File = "/configs/cluster.hocon",
assert_config_load_done(Nodes),
rpc:call(First, ?MODULE, create_data_dir, []),
rpc:call(First, ?MODULE, create_data_dir, [File]),
{[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]),
{[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []),
ok = rpc:call(First, application, start, [emqx_conf]),
{[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]),
assert_data_copy_done(Nodes),
assert_data_copy_done(Nodes, File),
stop_cluster(Nodes),
ok
after
stop_cluster(Nodes)
end.
t_copy_deprecated_data_dir(_Config) ->
net_kernel:start(['master2@127.0.0.1', longnames]),
ct:timetrap({seconds, 120}),
snabbkaffe:fix_ct_logging(),
Cluster = cluster([cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})]),
%% 1. Start all nodes
[First | Rest] = Nodes = start_cluster(Cluster),
try
File = "/configs/cluster-override.conf",
assert_config_load_done(Nodes),
rpc:call(First, ?MODULE, create_data_dir, [File]),
{[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]),
{[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []),
ok = rpc:call(First, application, start, [emqx_conf]),
{[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]),
assert_data_copy_done(Nodes, File),
stop_cluster(Nodes),
ok
after
@ -77,7 +102,7 @@ t_copy_data_dir(_Config) ->
%% Helper functions
%%------------------------------------------------------------------------------
create_data_dir() ->
create_data_dir(File) ->
Node = atom_to_list(node()),
ok = filelib:ensure_dir(Node ++ "/certs/"),
ok = filelib:ensure_dir(Node ++ "/authz/"),
@ -85,7 +110,7 @@ create_data_dir() ->
ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)),
ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)),
Telemetry = <<"telemetry.enable = false">>,
ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry).
ok = file:write_file(Node ++ File, Telemetry).
set_data_dir_env() ->
Node = atom_to_list(node()),
@ -100,14 +125,17 @@ set_data_dir_env() ->
ok = file:write_file(NewConfigFile, DataDir, [append]),
application:set_env(emqx, config_files, [NewConfigFile]),
application:set_env(emqx, data_dir, Node),
%% We set env both cluster.hocon and cluster-override.conf, but only one will be used
application:set_env(emqx, cluster_hocon_file, Node ++ "/configs/cluster.hocon"),
application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"),
ok.
assert_data_copy_done([First0 | Rest]) ->
assert_data_copy_done([First0 | Rest], File) ->
First = atom_to_list(First0),
{ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"),
{ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"),
{ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster-override.conf"),
{ok, FakeOverrideFile} = file:read_file(First ++ File),
{ok, ExpectFake} = hocon:binary(FakeOverrideFile),
lists:foreach(
fun(Node0) ->
Node = atom_to_list(Node0),
@ -117,8 +145,8 @@ assert_data_copy_done([First0 | Rest]) ->
#{node => Node}
),
?assertEqual(
{ok, FakeOverrideFile},
file:read_file(Node ++ "/configs/cluster-override.conf"),
{ok, ExpectFake},
hocon:files([Node ++ File]),
#{node => Node}
),
?assertEqual(
@ -173,3 +201,6 @@ cluster(Specs) ->
ok
end}
]).
cluster_spec({Type, Num}) ->
{Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}.

138
apps/emqx_management/src/emqx_mgmt_api_configs.erl
View File

@ -31,41 +31,23 @@
global_zone_configs/3
]).
-export([gen_schema/1]).
-define(PREFIX, "/configs/").
-define(PREFIX_RESET, "/configs_reset/").
-define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))).
-define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}).
-define(TAGS, ["Configs"]).
-define(EXCLUDES,
[
<<"exhook">>,
<<"gateway">>,
<<"plugins">>,
<<"bridges">>,
<<"rule_engine">>,
<<"authorization">>,
<<"authentication">>,
<<"rpc">>,
<<"connectors">>,
<<"slow_subs">>,
<<"psk_authentication">>,
<<"topic_metrics">>,
<<"rewrite">>,
<<"auto_subscribe">>,
<<"retainer">>,
<<"statsd">>,
<<"delayed">>,
<<"event_message">>,
<<"prometheus">>,
<<"telemetry">>,
<<"listeners">>,
<<"license">>,
<<"api_key">>
] ++ global_zone_roots()
).
-define(ROOT_KEYS, [
<<"dashboard">>,
<<"alarm">>,
<<"sys_topics">>,
<<"sysmon">>,
<<"limiter">>,
<<"trace">>,
<<"log">>,
<<"persistent_session_store">>,
<<"zones">>
]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@ -119,7 +101,6 @@ schema("/configs_reset/:rootname") ->
"- For a config entry that has default value, this resets it to the default value;\n"
"- For a config entry that has no default value, an error 400 will be returned"
>>,
summary => <<"Reset config entry">>,
%% We only return "200" rather than the new configs that has been changed, as
%% the schema of the changed configs is depends on the request parameter
%% `conf_path`, it cannot be defined here.
@ -214,12 +195,11 @@ fields(Field) ->
%%%==============================================================================================
%% HTTP API Callbacks
config(get, _Params, Req) ->
[Path] = conf_path(Req),
{200, get_raw_config(Path)};
config(put, #{body := NewConf}, Req) ->
Path = conf_path(Req),
{ok, Conf} = emqx_utils_maps:deep_find(Path, get_full_config()),
{200, Conf};
config(put, #{body := Body}, Req) ->
Path = conf_path(Req),
case emqx_conf:update(Path, Body, ?OPTS) of
case emqx_conf:update(Path, NewConf, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
{200, RawConf};
{error, {permission_denied, Reason}} ->
@ -229,28 +209,29 @@ config(put, #{body := Body}, Req) ->
end.
global_zone_configs(get, _Params, _Req) ->
Paths = global_zone_roots(),
Zones = lists:foldl(
fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end,
#{},
Paths
),
{200, Zones};
{200, get_zones()};
global_zone_configs(put, #{body := Body}, _Req) ->
PrevZones = get_zones(),
Res =
maps:fold(
fun(Path, Value, Acc) ->
case emqx_conf:update([Path], Value, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
Acc#{Path => RawConf};
{error, Reason} ->
?SLOG(error, #{
msg => "update global zone failed",
reason => Reason,
path => Path,
value => Value
}),
Acc
PrevValue = maps:get(Path, PrevZones),
case Value =/= PrevValue of
true ->
case emqx_conf:update([Path], Value, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
Acc#{Path => RawConf};
{error, Reason} ->
?SLOG(error, #{
msg => "update global zone failed",
reason => Reason,
path => Path,
value => Value
}),
Acc
end;
false ->
Acc#{Path => Value}
end
end,
#{},
@ -298,13 +279,30 @@ conf_path_reset(Req) ->
get_full_config() ->
emqx_config:fill_defaults(
maps:without(
?EXCLUDES,
maps:with(
?ROOT_KEYS,
emqx:get_raw_config([])
),
#{obfuscate_sensitive_values => true}
).
get_raw_config(Path) ->
#{Path := Conf} =
emqx_config:fill_defaults(
#{Path => emqx:get_raw_config([Path])},
#{obfuscate_sensitive_values => true}
),
Conf.
get_zones() ->
lists:foldl(
fun(Path, Acc) ->
maps:merge(Acc, get_config_with_default(Path))
end,
#{},
global_zone_roots()
).
get_config_with_default(Path) ->
emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}).
@ -317,40 +315,12 @@ conf_path_from_querystr(Req) ->
config_list() ->
Mod = emqx_conf:schema_module(),
Roots = hocon_schema:roots(Mod),
lists:foldl(fun(Key, Acc) -> lists:keydelete(Key, 1, Acc) end, Roots, ?EXCLUDES).
lists:foldl(fun(Key, Acc) -> [lists:keyfind(Key, 1, Roots) | Acc] end, [], ?ROOT_KEYS).
conf_path(Req) ->
<<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req),
string:lexemes(Path, "/ ").
%% TODO: generate from hocon schema
gen_schema(Conf) when is_boolean(Conf) ->
with_default_value(#{type => boolean}, Conf);
gen_schema(Conf) when is_binary(Conf); is_atom(Conf) ->
with_default_value(#{type => string}, Conf);
gen_schema(Conf) when is_number(Conf) ->
with_default_value(#{type => number}, Conf);
gen_schema(Conf) when is_list(Conf) ->
case io_lib:printable_unicode_list(Conf) of
true ->
gen_schema(unicode:characters_to_binary(Conf));
false ->
#{type => array, items => gen_schema(hd(Conf))}
end;
gen_schema(Conf) when is_map(Conf) ->
#{
type => object,
properties =>
maps:map(fun(_K, V) -> gen_schema(V) end, Conf)
};
gen_schema(_Conf) ->
%% the conf is not of JSON supported type, it may have been converted
%% by the hocon schema
#{type => string}.
with_default_value(Type, Value) ->
Type#{example => emqx_utils_maps:binary_string(Value)}.
global_zone_roots() ->
lists:map(fun({K, _}) -> list_to_binary(K) end, global_zone_schema()).

62
apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
View File

@ -55,12 +55,17 @@ t_update(_Config) ->
%% update ok
{ok, SysMon} = get_config(<<"sysmon">>),
#{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon,
NewSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort),
NewSysMon = #{<<"vm">> => #{<<"busy_port">> => not BusyPort}},
{ok, #{}} = update_config(<<"sysmon">>, NewSysMon),
{ok, SysMon1} = get_config(<<"sysmon">>),
#{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1,
?assertEqual(BusyPort, not BusyPort1),
assert_busy_port(BusyPort1),
%% Make sure the override config is updated, and remove the default value.
?assertMatch(
#{<<"vm">> := #{<<"busy_port">> := BusyPort1}},
maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster}))
),
%% update failed
ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"),
@ -138,6 +143,8 @@ t_global_zone(_Config) ->
NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
{ok, #{}} = update_global_zone(NewZones),
?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])),
%% Make sure the override config is updated, and remove the default value.
?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)),
BadZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3),
?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)),
@ -153,6 +160,19 @@ t_global_zone(_Config) ->
%% the default value is 2
?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)),
ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0),
DefaultZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2),
{ok, #{}} = update_global_zone(DefaultZones),
#{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}),
Default = maps:map(
fun
(_, V) when is_boolean(V) -> V;
(_, V) when is_atom(V) -> atom_to_binary(V);
(_, V) -> V
end,
Mqtt
),
?assertEqual(Default, read_conf(<<"mqtt">>)),
ok.
get_global_zone() ->
@ -177,7 +197,7 @@ t_dashboard(_Config) ->
Https1 = #{enable => true, bind => 18084},
?assertMatch(
{error, {"HTTP/1.1", 400, _}},
update_config("dashboard", Dashboard#{<<"https">> => Https1})
update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})
),
Https2 = #{
@ -187,35 +207,41 @@ t_dashboard(_Config) ->
cacertfile => "etc/certs/badcacert.pem",
certfile => "etc/certs/badcert.pem"
},
Dashboard2 = Dashboard#{listeners => Listeners#{https => Https2}},
Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}},
?assertMatch(
{error, {"HTTP/1.1", 400, _}},
update_config("dashboard", Dashboard2)
),
Keyfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])),
Certfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])),
Cacertfile = emqx_common_test_helpers:app_path(
KeyFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])),
CertFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])),
CacertFile = emqx_common_test_helpers:app_path(
emqx, filename:join(["etc", "certs", "cacert.pem"])
),
Https3 = #{
enable => true,
bind => 18084,
keyfile => Keyfile,
cacertfile => Cacertfile,
certfile => Certfile
<<"enable">> => true,
<<"bind">> => 18084,
<<"keyfile">> => list_to_binary(KeyFile),
<<"cacertfile">> => list_to_binary(CacertFile),
<<"certfile">> => list_to_binary(CertFile)
},
Dashboard3 = Dashboard#{listeners => Listeners#{https => Https3}},
Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}},
?assertMatch({ok, _}, update_config("dashboard", Dashboard3)),
Dashboard4 = Dashboard#{listeners => Listeners#{https => #{enable => false}}},
Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"enable">> => false}}},
?assertMatch({ok, _}, update_config("dashboard", Dashboard4)),
{ok, Dashboard41} = get_config("dashboard"),
?assertEqual(
Https3#{<<"enable">> => false},
read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]),
Dashboard41
),
?assertMatch({ok, _}, update_config("dashboard", Dashboard)),
{ok, Dashboard1} = get_config("dashboard"),
?assertNotEqual(Dashboard, Dashboard1),
timer:sleep(1000),
timer:sleep(1500),
ok.
t_configs_node({'init', Config}) ->
@ -296,3 +322,11 @@ reset_config(Name, Key) ->
{ok, []} -> ok;
Error -> Error
end.
read_conf(RootKeys) when is_list(RootKeys) ->
case emqx_config:read_override_conf(#{override_to => cluster}) of
undefined -> undefined;
Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined)
end;
read_conf(RootKey) ->
read_conf([RootKey]).

2
apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View File

@ -686,7 +686,7 @@ t_jq(_) ->
%% Got timeout as expected
got_timeout
end,
ConfigRootKey = emqx_rule_engine_schema:namespace(),
_ConfigRootKey = emqx_rule_engine_schema:namespace(),
?assertThrow(
{jq_exception, {timeout, _}},
apply_func(jq, [TOProgram, <<"-2">>])

7
changes/ce/feat-10156.en.md Normal file
View File

@ -0,0 +1,7 @@
Change the priority of the configuration:
1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`.
2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`.
Deprecated data/configs/local-override.conf.
Stabilizing the HTTP API for hot updates.

2
mix.exs
View File

@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true},
{:hocon, github: "emqx/hocon", tag: "0.38.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.38.1", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

2
rebar.config
View File

@ -75,7 +75,7 @@
, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}