yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10984 from zhongwencool/reload-cli 

feat: support reload cli
This commit is contained in:
zhongwencool 2023-06-12 17:21:29 +08:00 committed by GitHub
parent 12b73ba1c0 072f6fac8d
commit 0fc10ad239
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 563 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/emqx/include/emqx_authentication.hrl
View File

@ -47,5 +47,6 @@
-define(CMD_MOVE_REAR, rear).
-define(CMD_MOVE_BEFORE(Before), {before, Before}).
-define(CMD_MOVE_AFTER(After), {'after', After}).
-define(CMD_MERGE, merge).
-endif.

2
apps/emqx/rebar.config
View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

81
apps/emqx/src/emqx_authentication_config.erl
View File

@ -55,7 +55,9 @@
{create_authenticator, chain_name(), map()}
| {delete_authenticator, chain_name(), authenticator_id()}
| {update_authenticator, chain_name(), authenticator_id(), map()}
| {move_authenticator, chain_name(), authenticator_id(), position()}.
| {move_authenticator, chain_name(), authenticator_id(), position()}
| {merge_authenticators, map()}
| map().
%%------------------------------------------------------------------------------
%% Callbacks of config handler
@ -128,6 +130,9 @@ do_pre_config_update(_, {move_authenticator, _ChainName, AuthenticatorID, Positi
end
end
end;
do_pre_config_update(Paths, {merge_authenticators, NewConfig}, OldConfig) ->
MergeConfig = merge_authenticators(OldConfig, NewConfig),
do_pre_config_update(Paths, MergeConfig, OldConfig);
do_pre_config_update(_, OldConfig, OldConfig) ->
{ok, OldConfig};
do_pre_config_update(Paths, NewConfig, _OldConfig) ->
@ -327,3 +332,77 @@ chain_name([authentication]) ->
?GLOBAL;
chain_name([listeners, Type, Name, authentication]) ->
binary_to_existing_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>).
merge_authenticators(OriginConf0, NewConf0) ->
{OriginConf1, NewConf1} =
lists:foldl(
fun(Origin, {OriginAcc, NewAcc}) ->
AuthenticatorID = authenticator_id(Origin),
case split_by_id(AuthenticatorID, NewAcc) of
{error, _} ->
{[Origin | OriginAcc], NewAcc};
{ok, BeforeFound, [Found | AfterFound]} ->
Merged = emqx_utils_maps:deep_merge(Origin, Found),
{[Merged | OriginAcc], BeforeFound ++ AfterFound}
end
end,
{[], NewConf0},
OriginConf0
),
lists:reverse(OriginConf1) ++ NewConf1.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-compile(nowarn_export_all).
-compile(export_all).
merge_authenticators_test() ->
?assertEqual([], merge_authenticators([], [])),
Http = #{
<<"mechanism">> => <<"password_based">>, <<"backend">> => <<"http">>, <<"enable">> => true
},
Jwt = #{<<"mechanism">> => <<"jwt">>, <<"enable">> => true},
BuildIn = #{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"built_in_database">>,
<<"enable">> => true
},
Mongodb = #{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"mongodb">>,
<<"enable">> => true
},
Redis = #{
<<"mechanism">> => <<"password_based">>, <<"backend">> => <<"redis">>, <<"enable">> => true
},
BuildInDisable = BuildIn#{<<"enable">> => false},
MongodbDisable = Mongodb#{<<"enable">> => false},
RedisDisable = Redis#{<<"enable">> => false},
%% add
?assertEqual([Http], merge_authenticators([], [Http])),
?assertEqual([Http, Jwt, BuildIn], merge_authenticators([Http], [Jwt, BuildIn])),
%% merge
?assertEqual(
[BuildInDisable, MongodbDisable],
merge_authenticators([BuildIn, Mongodb], [BuildInDisable, MongodbDisable])
),
?assertEqual(
[BuildInDisable, Jwt],
merge_authenticators([BuildIn, Jwt], [BuildInDisable])
),
?assertEqual(
[BuildInDisable, Jwt, Mongodb],
merge_authenticators([BuildIn, Jwt], [Mongodb, BuildInDisable])
),
%% position changed
?assertEqual(
[BuildInDisable, Jwt, Mongodb, RedisDisable, Http],
merge_authenticators([BuildIn, Jwt, Mongodb, Redis], [RedisDisable, BuildInDisable, Http])
),
ok.
-endif.

12
apps/emqx/src/emqx_config.erl
View File

@ -18,6 +18,7 @@
-compile({no_auto_import, [get/0, get/1, put/2, erase/1]}).
-elvis([{elvis_style, god_modules, disable}]).
-include("logger.hrl").
-include("emqx.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-export([
@ -33,7 +34,9 @@
save_configs/5,
save_to_app_env/1,
save_to_config_map/2,
save_to_override_conf/3
save_to_override_conf/3,
config_files/0,
include_dirs/0
]).
-export([merge_envs/2]).
@ -89,6 +92,7 @@
]).
-export([ensure_atom_conf_path/2]).
-export([load_config_files/2]).
-ifdef(TEST).
-export([erase_all/0, backup_and_write/2]).
@ -311,8 +315,7 @@ put_raw(KeyPath0, Config) ->
%% Load/Update configs From/To files
%%============================================================================
init_load(SchemaMod) ->
ConfFiles = application:get_env(emqx, config_files, []),
init_load(SchemaMod, ConfFiles).
init_load(SchemaMod, config_files()).
%% @doc Initial load of the given config files.
%% NOTE: The order of the files is significant, configs from files ordered
@ -977,3 +980,6 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
ok;
put_config_post_change_actions(_Key, _NewValue) ->
ok.
config_files() ->
application:get_env(emqx, config_files, []).

10
apps/emqx/src/emqx_config_handler.erl
View File

@ -44,11 +44,12 @@
code_change/3
]).
-define(MOD, {mod}).
-export([schema/2]).
-define(MOD, '$mod').
-define(WKEY, '?').
-type handler_name() :: module().
-type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}.
-optional_callbacks([
pre_config_update/3,
@ -67,10 +68,7 @@
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
-type state() :: #{
handlers := handlers(),
atom() => term()
}.
-type state() :: #{handlers := any()}.
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []).

4
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -286,9 +286,9 @@ perform_sanity_checks(_App) ->
ok.
ensure_config_handler(Module, ConfigPath) ->
#{handlers := Handlers} = sys:get_state(emqx_config_handler),
#{handlers := Handlers} = emqx_config_handler:info(),
case emqx_utils_maps:deep_get(ConfigPath, Handlers, not_found) of
#{{mod} := Module} -> ok;
#{'$mod' := Module} -> ok;
NotFound -> error({config_handler_missing, ConfigPath, Module, NotFound})
end,
ok.

6
apps/emqx/test/emqx_config_SUITE.erl
View File

@ -63,12 +63,12 @@ t_fill_default_values(C) when is_list(C) ->
<<"enable_session_registry">> := true,
<<"perf">> :=
#{
<<"route_lock_type">> := key,
<<"route_lock_type">> := <<"key">>,
<<"trie_compaction">> := true
},
<<"route_batch_clean">> := false,
<<"session_locking_strategy">> := quorum,
<<"shared_subscription_strategy">> := round_robin
<<"session_locking_strategy">> := <<"quorum">>,
<<"shared_subscription_strategy">> := <<"round_robin">>
}
},
WithDefaults

4
apps/emqx/test/emqx_config_handler_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(export_all).
-compile(nowarn_export_all).
-define(MOD, {mod}).
-define(MOD, '$mod').
-define(WKEY, '?').
-define(CLUSTER_CONF, "/tmp/cluster.conf").
@ -99,7 +99,7 @@ t_conflict_handler(_Config) ->
%% override
ok = emqx_config_handler:add_handler([sysmon], emqx_config_logger),
?assertMatch(
#{handlers := #{sysmon := #{{mod} := emqx_config_logger}}},
#{handlers := #{sysmon := #{?MOD := emqx_config_logger}}},
emqx_config_handler:info()
),
ok.

8
apps/emqx_authn/src/emqx_authn.erl
View File

@ -26,10 +26,7 @@
get_enabled_authns/0
]).
%% Data backup
-export([
import_config/1
]).
-export([merge_config/1, import_config/1]).
-include("emqx_authn.hrl").
@ -162,3 +159,6 @@ authn_list(Authn) when is_list(Authn) ->
Authn;
authn_list(Authn) when is_map(Authn) ->
[Authn].
merge_config(AuthNs) ->
emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}).

2
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -89,6 +89,8 @@
param_listener_id/0
]).
-export([update_config/2]).
-elvis([{elvis_style, god_modules, disable}]).
api_spec() ->

2
apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
View File

@ -451,7 +451,7 @@ request_for_log(Credential, #{url := Url, method := Method} = State) ->
base_url => Url,
path_query => PathQuery,
headers => Headers,
mody => Body
body => Body
}
end.

6
apps/emqx_authz/etc/emqx_authz.conf
View File

@ -1,5 +1 @@
authorization {
deny_action = ignore
no_match = allow
cache = { enable = true }
}

1
apps/emqx_authz/include/emqx_authz.hrl
View File

@ -37,6 +37,7 @@
-define(CMD_PREPEND, prepend).
-define(CMD_APPEND, append).
-define(CMD_MOVE, move).
-define(CMD_MERGE, merge).
-define(CMD_MOVE_FRONT, front).
-define(CMD_MOVE_REAR, rear).

115
apps/emqx_authz/src/emqx_authz.erl
View File

@ -24,11 +24,6 @@
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-ifdef(TEST).
-compile(export_all).
-compile(nowarn_export_all).
-endif.
-export([
register_metrics/0,
init/0,
@ -37,6 +32,7 @@
lookup/1,
move/2,
update/2,
merge/1,
authorize/5,
%% for telemetry information
get_enabled_authzs/0
@ -128,6 +124,9 @@ lookup(Type) ->
{Source, _Front, _Rear} = take(Type),
Source.
merge(NewConf) ->
emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}).
move(Type, ?CMD_MOVE_BEFORE(Before)) ->
emqx_authz_utils:update_config(
?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}
@ -158,18 +157,25 @@ pre_config_update(Path, Cmd, Sources) ->
do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources) ->
do_pre_config_update(Cmd, Sources);
do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf) ->
do_pre_config_merge(NewConf, OldConf);
do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
do_pre_config_replace(NewConf, OldConf).
do_pre_config_merge(NewConf, OldConf) ->
MergeConf = emqx_utils_maps:deep_merge(OldConf, NewConf),
NewSources = merge_sources(OldConf, NewConf),
do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf).
%% override the entire config when updating the root key
%% emqx_conf:update(?ROOT_KEY, Conf);
do_pre_config_replace(Conf, Conf) ->
Conf;
do_pre_config_replace(NewConf, OldConf) ->
#{<<"sources">> := NewSources} = NewConf,
#{<<"sources">> := OldSources} = OldConf,
NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
NewConf#{<<"sources">> := NewSources1}.
NewSources = get_sources(NewConf),
OldSources = get_sources(OldConf),
ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
NewConf#{<<"sources">> => ReplaceSources}.
do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) ->
do_move(Cmd, Sources);
@ -465,8 +471,8 @@ get_enabled_authzs() ->
%%------------------------------------------------------------------------------
import_config(#{?CONF_NS_BINARY := AuthzConf}) ->
Sources = maps:get(<<"sources">>, AuthzConf, []),
OldSources = emqx:get_raw_config(?CONF_KEY_PATH, []),
Sources = get_sources(AuthzConf),
OldSources = emqx:get_raw_config(?CONF_KEY_PATH, [emqx_authz_schema:default_authz()]),
MergedSources = emqx_utils:merge_lists(OldSources, Sources, fun type/1),
MergedAuthzConf = AuthzConf#{<<"sources">> => MergedSources},
case emqx_conf:update([?CONF_NS_ATOM], MergedAuthzConf, #{override_to => cluster}) of
@ -526,12 +532,12 @@ take(Type) -> take(Type, lookup()).
%% Take the source of give type, the sources list is split into two parts
%% front part and rear part.
take(Type, Sources) ->
{Front, Rear} = lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources),
case Rear =:= [] of
true ->
Expect = type(Type),
case lists:splitwith(fun(T) -> type(T) =/= Expect end, Sources) of
{_Front, []} ->
throw({not_found_source, Type});
_ ->
{hd(Rear), Front, tl(Rear)}
{Front, [Found | Rear]} ->
{Found, Front, Rear}
end.
find_action_in_hooks() ->
@ -628,3 +634,80 @@ check_acl_file_rules(Path, Rules) ->
after
_ = file:delete(TmpPath)
end.
merge_sources(OriginConf, NewConf) ->
{OriginSource, NewSources} =
lists:foldl(
fun(Old = #{<<"type">> := Type}, {OriginAcc, NewAcc}) ->
case type_take(Type, NewAcc) of
not_found ->
{[Old | OriginAcc], NewAcc};
{New, NewAcc1} ->
MergeSource = emqx_utils_maps:deep_merge(Old, New),
{[MergeSource | OriginAcc], NewAcc1}
end
end,
{[], get_sources(NewConf)},
get_sources(OriginConf)
),
lists:reverse(OriginSource) ++ NewSources.
get_sources(Conf) ->
Default = [emqx_authz_schema:default_authz()],
maps:get(<<"sources">>, Conf, Default).
type_take(Type, Sources) ->
try take(Type, Sources) of
{Found, Front, Rear} -> {Found, Front ++ Rear}
catch
throw:{not_found_source, Type} -> not_found
end.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-compile(nowarn_export_all).
-compile(export_all).
merge_sources_test() ->
Default = [emqx_authz_schema:default_authz()],
Http = #{<<"type">> => <<"http">>, <<"enable">> => true},
Mysql = #{<<"type">> => <<"mysql">>, <<"enable">> => true},
Mongo = #{<<"type">> => <<"mongodb">>, <<"enable">> => true},
Redis = #{<<"type">> => <<"redis">>, <<"enable">> => true},
Postgresql = #{<<"type">> => <<"postgresql">>, <<"enable">> => true},
HttpDisable = Http#{<<"enable">> => false},
MysqlDisable = Mysql#{<<"enable">> => false},
MongoDisable = Mongo#{<<"enable">> => false},
%% has default source
?assertEqual(Default, merge_sources(#{}, #{})),
?assertEqual([], merge_sources(#{<<"sources">> => []}, #{<<"sources">> => []})),
?assertEqual(Default, merge_sources(#{}, #{<<"sources">> => []})),
%% add
?assertEqual(
[Http, Mysql, Mongo, Redis, Postgresql],
merge_sources(
#{<<"sources">> => [Http, Mysql]},
#{<<"sources">> => [Mongo, Redis, Postgresql]}
)
),
%% replace
?assertEqual(
[HttpDisable, MysqlDisable],
merge_sources(
#{<<"sources">> => [Http, Mysql]},
#{<<"sources">> => [HttpDisable, MysqlDisable]}
)
),
%% add + replace + change position
?assertEqual(
[HttpDisable, Mysql, MongoDisable, Redis],
merge_sources(
#{<<"sources">> => [Http, Mysql, Mongo]},
#{<<"sources">> => [MongoDisable, HttpDisable, Redis]}
)
),
ok.
-endif.

3
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -42,7 +42,8 @@
-export([
headers_no_content_type/1,
headers/1
headers/1,
default_authz/0
]).
%%--------------------------------------------------------------------

2
apps/emqx_conf/include/emqx_conf.hrl
View File

@ -34,4 +34,6 @@
tnx_id :: pos_integer() | '$1'
}).
-define(READONLY_KEYS, [cluster, rpc, node]).
-endif.

11
apps/emqx_conf/src/emqx_conf.erl
View File

@ -19,6 +19,7 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx_schema.hrl").
-include("emqx_conf.hrl").
-export([add_handler/2, remove_handler/1]).
-export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]).
@ -30,6 +31,7 @@
-export([dump_schema/2]).
-export([schema_module/0]).
-export([gen_example_conf/2]).
-export([check_config/2]).
%% TODO: move to emqx_dashboard when we stop building api schema at build time
-export([
@ -213,6 +215,15 @@ schema_module() ->
Value -> list_to_existing_atom(Value)
end.
check_config(Mod, Raw) ->
try
{_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw),
{ok, CheckedConf}
catch
throw:Error ->
{error, Error}
end.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------

212
apps/emqx_conf/src/emqx_conf_cli.erl
View File

@ -15,6 +15,10 @@
%%--------------------------------------------------------------------
-module(emqx_conf_cli).
-include("emqx_conf.hrl").
-include_lib("emqx/include/emqx_access_control.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-export([
load/0,
admins/1,
@ -27,6 +31,7 @@
%% kept cluster_call for compatibility
-define(CLUSTER_CALL, cluster_call).
-define(CONF, conf).
-define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited").
load() ->
emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]),
@ -42,10 +47,16 @@ conf(["show"]) ->
print_hocon(get_config());
conf(["show", Key]) ->
print_hocon(get_config(Key));
conf(["load", "--auth-chains", AuthChains, Path]) when
AuthChains =:= "replace"; AuthChains =:= "merge"
->
load_config(Path, AuthChains);
conf(["load", Path]) ->
load_config(Path);
load_config(Path, "replace");
conf(["cluster_sync" | Args]) ->
admins(Args);
conf(["reload"]) ->
reload_etc_conf_on_local_node();
conf(_) ->
emqx_ctl:usage(usage_conf() ++ usage_sync()).
@ -87,8 +98,7 @@ admins(_) ->
usage_conf() ->
[
%% TODO add reload
%{"conf reload", "reload etc/emqx.conf on local node"},
{"conf reload", "reload etc/emqx.conf on local node"},
{"conf show_keys", "Print all config keys"},
{"conf show [<key>]",
"Print in-use configs (including default values) under the given key. "
@ -138,11 +148,14 @@ print_keys(Config) ->
print(Json) ->
emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]).
print_hocon(Hocon) ->
emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]).
print_hocon(Hocon) when is_map(Hocon) ->
emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]);
print_hocon({error, Error}) ->
emqx_ctl:warning("~ts~n", [Error]).
get_config() ->
drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))).
AllConf = emqx_config:fill_defaults(emqx:get_raw_config([])),
drop_hidden_roots(AllConf).
drop_hidden_roots(Conf) ->
Hidden = hidden_roots(),
@ -164,22 +177,183 @@ hidden_roots() ->
).
get_config(Key) ->
emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}).
case emqx:get_raw_config([Key], undefined) of
undefined -> {error, "key_not_found"};
Value -> emqx_config:fill_defaults(#{Key => Value})
end.
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
load_config(Path) ->
load_config(Path, AuthChain) ->
case hocon:files([Path]) of
{ok, Conf} ->
maps:foreach(
fun(Key, Value) ->
case emqx_conf:update([Key], Value, ?OPTIONS) of
{ok, _} -> emqx_ctl:print("load ~ts ok~n", [Key]);
{error, Reason} -> emqx_ctl:print("load ~ts failed: ~p~n", [Key, Reason])
end
end,
Conf
);
{ok, RawConf} when RawConf =:= #{} ->
emqx_ctl:warning("load ~ts is empty~n", [Path]),
{error, empty_hocon_file};
{ok, RawConf} ->
case check_config(RawConf) of
ok ->
lists:foreach(
fun({K, V}) -> update_config(K, V, AuthChain) end,
to_sorted_list(RawConf)
);
{error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]),
emqx_ctl:warning(
"Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n"
),
{error, Reason};
{error, Errors} ->
emqx_ctl:warning("load ~ts schema check failed~n", [Path]),
lists:foreach(
fun({Key, Error}) ->
emqx_ctl:warning("~ts: ~p~n", [Key, Error])
end,
Errors
),
{error, Errors}
end;
{error, Reason} ->
emqx_ctl:print("load ~ts failed~n~p~n", [Path, Reason]),
emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]),
{error, bad_hocon_file}
end.
update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
check_res(Key, emqx_authz:merge(Conf));
update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") ->
check_res(Key, emqx_authn:merge_config(Conf));
update_config(Key, Value, _) ->
check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)).
check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]);
check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]).
check_config(Conf) ->
case check_keys_is_not_readonly(Conf) of
ok -> check_config_schema(Conf);
Error -> Error
end.
check_keys_is_not_readonly(Conf) ->
Keys = maps:keys(Conf),
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
case ReadOnlyKeys -- Keys of
ReadOnlyKeys -> ok;
_ -> {error, ?UPDATE_READONLY_KEYS_PROHIBITED}
end.
check_config_schema(Conf) ->
SchemaMod = emqx_conf:schema_module(),
Fold = fun({Key, Value}, Acc) ->
Schema = emqx_config_handler:schema(SchemaMod, [Key]),
case emqx_conf:check_config(Schema, #{Key => Value}) of
{ok, _} -> Acc;
{error, Reason} -> [{Key, Reason} | Acc]
end
end,
sorted_fold(Fold, Conf).
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
reload_etc_conf_on_local_node() ->
case load_etc_config_file() of
{ok, RawConf} ->
case check_readonly_config(RawConf) of
{ok, Reloaded} -> reload_config(Reloaded);
{error, Error} -> {error, Error}
end;
{error, _Error} ->
{error, bad_hocon_file}
end.
%% @doc Merge etc/emqx.conf on top of cluster.hocon.
%% For example:
%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
%% but `authorization.sources` in not in the default emqx.conf file.
%% To make sure all root keys in emqx.conf has a fully merged value.
load_etc_config_file() ->
ConfFiles = emqx_config:config_files(),
Opts = #{format => map, include_dirs => emqx_config:include_dirs()},
case hocon:files(ConfFiles, Opts) of
{ok, RawConf} ->
HasDeprecatedFile = emqx_config:has_deprecated_file(),
%% Merge etc.conf on top of cluster.hocon,
%% Don't use map deep_merge, use hocon files merge instead.
%% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
Keys = maps:keys(RawConf),
MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles),
{ok, maps:with(Keys, MergedRaw)};
{error, Error} ->
?SLOG(error, #{
msg => "failed_to_read_etc_config",
files => ConfFiles,
error => Error
}),
{error, Error}
end.
check_readonly_config(Raw) ->
SchemaMod = emqx_conf:schema_module(),
RawDefault = emqx_config:fill_defaults(Raw),
case emqx_conf:check_config(SchemaMod, RawDefault) of
{ok, CheckedConf} ->
case filter_changed_readonly_keys(CheckedConf) of
[] ->
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
{ok, maps:without(ReadOnlyKeys, Raw)};
Error ->
?SLOG(error, #{
msg => ?UPDATE_READONLY_KEYS_PROHIBITED,
read_only_keys => ?READONLY_KEYS,
error => Error
}),
{error, Error}
end;
{error, Error} ->
?SLOG(error, #{
msg => "bad_etc_config_schema_found",
error => Error
}),
{error, Error}
end.
reload_config(AllConf) ->
Fold = fun({Key, Conf}, Acc) ->
case emqx:update_config([Key], Conf, #{persistent => false}) of
{ok, _} ->
emqx_ctl:print("Reloaded ~ts config ok~n", [Key]),
Acc;
Error ->
emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]),
?SLOG(error, #{
msg => "failed_to_reload_etc_config",
key => Key,
value => Conf,
error => Error
}),
[{Key, Error} | Acc]
end
end,
sorted_fold(Fold, AllConf).
filter_changed_readonly_keys(Conf) ->
lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS).
filter_changed(Key, ChangedConf) ->
Prev = emqx_conf:get([Key], #{}),
New = maps:get(Key, ChangedConf, #{}),
case Prev =/= New of
true -> {true, {Key, changed(New, Prev)}};
false -> false
end.
changed(New, Prev) ->
Diff = emqx_utils_maps:diff_maps(New, Prev),
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
sorted_fold(Func, Conf) ->
case lists:foldl(Func, [], to_sorted_list(Conf)) of
[] -> ok;
Error -> {error, Error}
end.
to_sorted_list(Conf) ->
lists:keysort(1, maps:to_list(Conf)).

131
apps/emqx_conf/test/emqx_conf_cli_SUITE.erl Normal file
View File

@ -0,0 +1,131 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_conf_cli_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_conf.hrl").
-import(emqx_config_SUITE, [prepare_conf_file/3]).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]),
Config.
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
t_load_config(Config) ->
Authz = authorization,
Conf = emqx_conf:get_raw([Authz]),
%% set sources to []
ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
ok = emqx_conf_cli:conf(["load", ConfFile0]),
?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])),
%% remove sources, it will reset to default file source.
ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
ok = emqx_conf_cli:conf(["load", ConfFile1]),
Default = [emqx_authz_schema:default_authz()],
?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])),
%% reset
ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
ok = emqx_conf_cli:conf(["load", ConfFile2]),
?assertEqual(
Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]},
emqx_conf:get_raw([Authz])
),
?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])),
ok.
t_load_readonly(Config) ->
Base0 = base_conf(),
Base1 = Base0#{<<"mqtt">> => emqx_conf:get_raw([mqtt])},
lists:foreach(
fun(Key) ->
KeyBin = atom_to_binary(Key),
Conf = emqx_conf:get_raw([Key]),
ConfBin0 = hocon_pp:do(Base1#{KeyBin => Conf}, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
?assertEqual(
{error, "update_readonly_keys_prohibited"},
emqx_conf_cli:conf(["load", ConfFile0])
),
%% reload etc/emqx.conf changed readonly keys
ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}),
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
application:set_env(emqx, config_files, [ConfFile1]),
?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"]))
end,
?READONLY_KEYS
),
ok.
t_error_schema_check(Config) ->
Base = #{
%% bad multiplier
<<"mqtt">> => #{<<"keepalive_multiplier">> => -1},
<<"zones">> => #{<<"my-zone">> => #{<<"mqtt">> => #{<<"keepalive_multiplier">> => 10}}}
},
ConfBin0 = hocon_pp:do(Base, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
?assertMatch({error, _}, emqx_conf_cli:conf(["load", ConfFile0])),
%% zones is not updated because of error
?assertEqual(#{}, emqx_config:get_raw([zones])),
ok.
t_reload_etc_emqx_conf_not_persistent(Config) ->
Mqtt = emqx_conf:get_raw([mqtt]),
Base = base_conf(),
Conf = Base#{<<"mqtt">> => Mqtt#{<<"keepalive_multiplier">> => 3}},
ConfBin = hocon_pp:do(Conf, #{}),
ConfFile = prepare_conf_file(?FUNCTION_NAME, ConfBin, Config),
application:set_env(emqx, config_files, [ConfFile]),
ok = emqx_conf_cli:conf(["reload"]),
?assertEqual(3, emqx:get_config([mqtt, keepalive_multiplier])),
?assertNotEqual(
3,
emqx_utils_maps:deep_get(
[<<"mqtt">>, <<"keepalive_multiplier">>],
emqx_config:read_override_conf(#{}),
undefined
)
),
ok.
base_conf() ->
#{
<<"cluster">> => emqx_conf:get_raw([cluster]),
<<"node">> => emqx_conf:get_raw([node])
}.
changed(cluster) ->
#{<<"name">> => <<"emqx-test">>};
changed(node) ->
#{
<<"name">> => <<"emqx-test@127.0.0.1">>,
<<"cookie">> => <<"gokdfkdkf1122">>,
<<"data_dir">> => <<"data">>
};
changed(rpc) ->
#{<<"mode">> => <<"sync">>}.

4
apps/emqx_conf/test/emqx_conf_logger_SUITE.erl
View File

@ -62,7 +62,7 @@ end_per_suite(_Config) ->
t_log_conf(_Conf) ->
FileExpect = #{
<<"enable">> => true,
<<"formatter">> => text,
<<"formatter">> => <<"text">>,
<<"level">> => <<"info">>,
<<"rotation_count">> => 10,
<<"rotation_size">> => <<"50MB">>,
@ -73,7 +73,7 @@ t_log_conf(_Conf) ->
<<"console">> =>
#{
<<"enable">> => true,
<<"formatter">> => text,
<<"formatter">> => <<"text">>,
<<"level">> => <<"debug">>,
<<"time_offset">> => <<"system">>
},

10
apps/emqx_ctl/src/emqx_ctl.erl
View File

@ -38,6 +38,8 @@
-export([
print/1,
print/2,
warning/1,
warning/2,
usage/1,
usage/2
]).
@ -180,6 +182,14 @@ print(Msg) ->
print(Format, Args) ->
io:format("~ts", [format(Format, Args)]).
-spec warning(io:format()) -> ok.
warning(Format) ->
warning(Format, []).
-spec warning(io:format(), [term()]) -> ok.
warning(Format, Args) ->
io:format("\e[31m~ts\e[0m", [format(Format, Args)]).
-spec usage([cmd_usage()]) -> ok.
usage(UsageList) ->
io:format(format_usage(UsageList)).

2
mix.exs
View File

@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
{:hocon, github: "emqx/hocon", tag: "0.39.7", override: true},
{:hocon, github: "emqx/hocon", tag: "0.39.8", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

2
rebar.config
View File

@ -75,7 +75,7 @@
, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}