diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl index 20ae2bf1e..70b35a474 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx/include/emqx_authentication.hrl @@ -47,5 +47,6 @@ -define(CMD_MOVE_REAR, rear). -define(CMD_MOVE_BEFORE(Before), {before, Before}). -define(CMD_MOVE_AFTER(After), {'after', After}). +-define(CMD_MERGE, merge). -endif. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9dedf3644..070922d22 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 92041095b..96718d611 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -55,7 +55,9 @@ {create_authenticator, chain_name(), map()} | {delete_authenticator, chain_name(), authenticator_id()} | {update_authenticator, chain_name(), authenticator_id(), map()} - | {move_authenticator, chain_name(), authenticator_id(), position()}. + | {move_authenticator, chain_name(), authenticator_id(), position()} + | {merge_authenticators, map()} + | map(). %%------------------------------------------------------------------------------ %% Callbacks of config handler @@ -128,6 +130,9 @@ do_pre_config_update(_, {move_authenticator, _ChainName, AuthenticatorID, Positi end end end; +do_pre_config_update(Paths, {merge_authenticators, NewConfig}, OldConfig) -> + MergeConfig = merge_authenticators(OldConfig, NewConfig), + do_pre_config_update(Paths, MergeConfig, OldConfig); do_pre_config_update(_, OldConfig, OldConfig) -> {ok, OldConfig}; do_pre_config_update(Paths, NewConfig, _OldConfig) -> @@ -327,3 +332,77 @@ chain_name([authentication]) -> ?GLOBAL; chain_name([listeners, Type, Name, authentication]) -> binary_to_existing_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>). + +merge_authenticators(OriginConf0, NewConf0) -> + {OriginConf1, NewConf1} = + lists:foldl( + fun(Origin, {OriginAcc, NewAcc}) -> + AuthenticatorID = authenticator_id(Origin), + case split_by_id(AuthenticatorID, NewAcc) of + {error, _} -> + {[Origin | OriginAcc], NewAcc}; + {ok, BeforeFound, [Found | AfterFound]} -> + Merged = emqx_utils_maps:deep_merge(Origin, Found), + {[Merged | OriginAcc], BeforeFound ++ AfterFound} + end + end, + {[], NewConf0}, + OriginConf0 + ), + lists:reverse(OriginConf1) ++ NewConf1. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-compile(nowarn_export_all). +-compile(export_all). + +merge_authenticators_test() -> + ?assertEqual([], merge_authenticators([], [])), + + Http = #{ + <<"mechanism">> => <<"password_based">>, <<"backend">> => <<"http">>, <<"enable">> => true + }, + Jwt = #{<<"mechanism">> => <<"jwt">>, <<"enable">> => true}, + BuildIn = #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"built_in_database">>, + <<"enable">> => true + }, + Mongodb = #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"mongodb">>, + <<"enable">> => true + }, + Redis = #{ + <<"mechanism">> => <<"password_based">>, <<"backend">> => <<"redis">>, <<"enable">> => true + }, + BuildInDisable = BuildIn#{<<"enable">> => false}, + MongodbDisable = Mongodb#{<<"enable">> => false}, + RedisDisable = Redis#{<<"enable">> => false}, + + %% add + ?assertEqual([Http], merge_authenticators([], [Http])), + ?assertEqual([Http, Jwt, BuildIn], merge_authenticators([Http], [Jwt, BuildIn])), + + %% merge + ?assertEqual( + [BuildInDisable, MongodbDisable], + merge_authenticators([BuildIn, Mongodb], [BuildInDisable, MongodbDisable]) + ), + ?assertEqual( + [BuildInDisable, Jwt], + merge_authenticators([BuildIn, Jwt], [BuildInDisable]) + ), + ?assertEqual( + [BuildInDisable, Jwt, Mongodb], + merge_authenticators([BuildIn, Jwt], [Mongodb, BuildInDisable]) + ), + + %% position changed + ?assertEqual( + [BuildInDisable, Jwt, Mongodb, RedisDisable, Http], + merge_authenticators([BuildIn, Jwt, Mongodb, Redis], [RedisDisable, BuildInDisable, Http]) + ), + ok. + +-endif. diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 835379fe7..f2f6eb264 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -18,6 +18,7 @@ -compile({no_auto_import, [get/0, get/1, put/2, erase/1]}). -elvis([{elvis_style, god_modules, disable}]). -include("logger.hrl"). +-include("emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ @@ -33,7 +34,9 @@ save_configs/5, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/3 + save_to_override_conf/3, + config_files/0, + include_dirs/0 ]). -export([merge_envs/2]). @@ -89,6 +92,7 @@ ]). -export([ensure_atom_conf_path/2]). +-export([load_config_files/2]). -ifdef(TEST). -export([erase_all/0, backup_and_write/2]). @@ -311,8 +315,7 @@ put_raw(KeyPath0, Config) -> %% Load/Update configs From/To files %%============================================================================ init_load(SchemaMod) -> - ConfFiles = application:get_env(emqx, config_files, []), - init_load(SchemaMod, ConfFiles). + init_load(SchemaMod, config_files()). %% @doc Initial load of the given config files. %% NOTE: The order of the files is significant, configs from files ordered @@ -977,3 +980,6 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) -> ok; put_config_post_change_actions(_Key, _NewValue) -> ok. + +config_files() -> + application:get_env(emqx, config_files, []). diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 1ca2e8b24..80c47d09e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -44,11 +44,12 @@ code_change/3 ]). --define(MOD, {mod}). +-export([schema/2]). + +-define(MOD, '$mod'). -define(WKEY, '?'). -type handler_name() :: module(). --type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}. -optional_callbacks([ pre_config_update/3, @@ -67,10 +68,7 @@ ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. --type state() :: #{ - handlers := handlers(), - atom() => term() -}. +-type state() :: #{handlers := any()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 1cfc10f74..bf4b2c0ad 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -286,9 +286,9 @@ perform_sanity_checks(_App) -> ok. ensure_config_handler(Module, ConfigPath) -> - #{handlers := Handlers} = sys:get_state(emqx_config_handler), + #{handlers := Handlers} = emqx_config_handler:info(), case emqx_utils_maps:deep_get(ConfigPath, Handlers, not_found) of - #{{mod} := Module} -> ok; + #{'$mod' := Module} -> ok; NotFound -> error({config_handler_missing, ConfigPath, Module, NotFound}) end, ok. diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index c87c2c533..5e9284a41 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -63,12 +63,12 @@ t_fill_default_values(C) when is_list(C) -> <<"enable_session_registry">> := true, <<"perf">> := #{ - <<"route_lock_type">> := key, + <<"route_lock_type">> := <<"key">>, <<"trie_compaction">> := true }, <<"route_batch_clean">> := false, - <<"session_locking_strategy">> := quorum, - <<"shared_subscription_strategy">> := round_robin + <<"session_locking_strategy">> := <<"quorum">>, + <<"shared_subscription_strategy">> := <<"round_robin">> } }, WithDefaults diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 194198571..d2f2faedb 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --define(MOD, {mod}). +-define(MOD, '$mod'). -define(WKEY, '?'). -define(CLUSTER_CONF, "/tmp/cluster.conf"). @@ -99,7 +99,7 @@ t_conflict_handler(_Config) -> %% override ok = emqx_config_handler:add_handler([sysmon], emqx_config_logger), ?assertMatch( - #{handlers := #{sysmon := #{{mod} := emqx_config_logger}}}, + #{handlers := #{sysmon := #{?MOD := emqx_config_logger}}}, emqx_config_handler:info() ), ok. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 2a8d82439..e717550f1 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -26,10 +26,7 @@ get_enabled_authns/0 ]). -%% Data backup --export([ - import_config/1 -]). +-export([merge_config/1, import_config/1]). -include("emqx_authn.hrl"). @@ -162,3 +159,6 @@ authn_list(Authn) when is_list(Authn) -> Authn; authn_list(Authn) when is_map(Authn) -> [Authn]. + +merge_config(AuthNs) -> + emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 65ce0cc32..b237108c2 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -89,6 +89,8 @@ param_listener_id/0 ]). +-export([update_config/2]). + -elvis([{elvis_style, god_modules, disable}]). api_spec() -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index f20632414..2f071a828 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -451,7 +451,7 @@ request_for_log(Credential, #{url := Url, method := Method} = State) -> base_url => Url, path_query => PathQuery, headers => Headers, - mody => Body + body => Body } end. diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 167b12b3f..8b1378917 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,5 +1 @@ -authorization { - deny_action = ignore - no_match = allow - cache = { enable = true } -} + diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 967865868..b43a2cdab 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -37,6 +37,7 @@ -define(CMD_PREPEND, prepend). -define(CMD_APPEND, append). -define(CMD_MOVE, move). +-define(CMD_MERGE, merge). -define(CMD_MOVE_FRONT, front). -define(CMD_MOVE_REAR, rear). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 3c9698de0..9fd6063e7 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -24,11 +24,6 @@ -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - -export([ register_metrics/0, init/0, @@ -37,6 +32,7 @@ lookup/1, move/2, update/2, + merge/1, authorize/5, %% for telemetry information get_enabled_authzs/0 @@ -128,6 +124,9 @@ lookup(Type) -> {Source, _Front, _Rear} = take(Type), Source. +merge(NewConf) -> + emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}). + move(Type, ?CMD_MOVE_BEFORE(Before)) -> emqx_authz_utils:update_config( ?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))} @@ -158,18 +157,25 @@ pre_config_update(Path, Cmd, Sources) -> do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources) -> do_pre_config_update(Cmd, Sources); +do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf) -> + do_pre_config_merge(NewConf, OldConf); do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(NewConf, OldConf). +do_pre_config_merge(NewConf, OldConf) -> + MergeConf = emqx_utils_maps:deep_merge(OldConf, NewConf), + NewSources = merge_sources(OldConf, NewConf), + do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf). + %% override the entire config when updating the root key %% emqx_conf:update(?ROOT_KEY, Conf); do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - #{<<"sources">> := NewSources} = NewConf, - #{<<"sources">> := OldSources} = OldConf, - NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), - NewConf#{<<"sources">> := NewSources1}. + NewSources = get_sources(NewConf), + OldSources = get_sources(OldConf), + ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), + NewConf#{<<"sources">> => ReplaceSources}. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); @@ -465,8 +471,8 @@ get_enabled_authzs() -> %%------------------------------------------------------------------------------ import_config(#{?CONF_NS_BINARY := AuthzConf}) -> - Sources = maps:get(<<"sources">>, AuthzConf, []), - OldSources = emqx:get_raw_config(?CONF_KEY_PATH, []), + Sources = get_sources(AuthzConf), + OldSources = emqx:get_raw_config(?CONF_KEY_PATH, [emqx_authz_schema:default_authz()]), MergedSources = emqx_utils:merge_lists(OldSources, Sources, fun type/1), MergedAuthzConf = AuthzConf#{<<"sources">> => MergedSources}, case emqx_conf:update([?CONF_NS_ATOM], MergedAuthzConf, #{override_to => cluster}) of @@ -526,12 +532,12 @@ take(Type) -> take(Type, lookup()). %% Take the source of give type, the sources list is split into two parts %% front part and rear part. take(Type, Sources) -> - {Front, Rear} = lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources), - case Rear =:= [] of - true -> + Expect = type(Type), + case lists:splitwith(fun(T) -> type(T) =/= Expect end, Sources) of + {_Front, []} -> throw({not_found_source, Type}); - _ -> - {hd(Rear), Front, tl(Rear)} + {Front, [Found | Rear]} -> + {Found, Front, Rear} end. find_action_in_hooks() -> @@ -628,3 +634,80 @@ check_acl_file_rules(Path, Rules) -> after _ = file:delete(TmpPath) end. + +merge_sources(OriginConf, NewConf) -> + {OriginSource, NewSources} = + lists:foldl( + fun(Old = #{<<"type">> := Type}, {OriginAcc, NewAcc}) -> + case type_take(Type, NewAcc) of + not_found -> + {[Old | OriginAcc], NewAcc}; + {New, NewAcc1} -> + MergeSource = emqx_utils_maps:deep_merge(Old, New), + {[MergeSource | OriginAcc], NewAcc1} + end + end, + {[], get_sources(NewConf)}, + get_sources(OriginConf) + ), + lists:reverse(OriginSource) ++ NewSources. + +get_sources(Conf) -> + Default = [emqx_authz_schema:default_authz()], + maps:get(<<"sources">>, Conf, Default). + +type_take(Type, Sources) -> + try take(Type, Sources) of + {Found, Front, Rear} -> {Found, Front ++ Rear} + catch + throw:{not_found_source, Type} -> not_found + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-compile(nowarn_export_all). +-compile(export_all). + +merge_sources_test() -> + Default = [emqx_authz_schema:default_authz()], + Http = #{<<"type">> => <<"http">>, <<"enable">> => true}, + Mysql = #{<<"type">> => <<"mysql">>, <<"enable">> => true}, + Mongo = #{<<"type">> => <<"mongodb">>, <<"enable">> => true}, + Redis = #{<<"type">> => <<"redis">>, <<"enable">> => true}, + Postgresql = #{<<"type">> => <<"postgresql">>, <<"enable">> => true}, + HttpDisable = Http#{<<"enable">> => false}, + MysqlDisable = Mysql#{<<"enable">> => false}, + MongoDisable = Mongo#{<<"enable">> => false}, + + %% has default source + ?assertEqual(Default, merge_sources(#{}, #{})), + ?assertEqual([], merge_sources(#{<<"sources">> => []}, #{<<"sources">> => []})), + ?assertEqual(Default, merge_sources(#{}, #{<<"sources">> => []})), + + %% add + ?assertEqual( + [Http, Mysql, Mongo, Redis, Postgresql], + merge_sources( + #{<<"sources">> => [Http, Mysql]}, + #{<<"sources">> => [Mongo, Redis, Postgresql]} + ) + ), + %% replace + ?assertEqual( + [HttpDisable, MysqlDisable], + merge_sources( + #{<<"sources">> => [Http, Mysql]}, + #{<<"sources">> => [HttpDisable, MysqlDisable]} + ) + ), + %% add + replace + change position + ?assertEqual( + [HttpDisable, Mysql, MongoDisable, Redis], + merge_sources( + #{<<"sources">> => [Http, Mysql, Mongo]}, + #{<<"sources">> => [MongoDisable, HttpDisable, Redis]} + ) + ), + ok. + +-endif. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 8e847b93e..b19c62441 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,7 +42,8 @@ -export([ headers_no_content_type/1, - headers/1 + headers/1, + default_authz/0 ]). %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index 0297fddf7..26042d62c 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -34,4 +34,6 @@ tnx_id :: pos_integer() | '$1' }). +-define(READONLY_KEYS, [cluster, rpc, node]). + -endif. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d2e45581d..584a10a8d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -19,6 +19,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/emqx_schema.hrl"). +-include("emqx_conf.hrl"). -export([add_handler/2, remove_handler/1]). -export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]). @@ -30,6 +31,7 @@ -export([dump_schema/2]). -export([schema_module/0]). -export([gen_example_conf/2]). +-export([check_config/2]). %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ @@ -213,6 +215,15 @@ schema_module() -> Value -> list_to_existing_atom(Value) end. +check_config(Mod, Raw) -> + try + {_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw), + {ok, CheckedConf} + catch + throw:Error -> + {error, Error} + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 530e4bfcb..f2aeec7fb 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -15,6 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_conf_cli). +-include("emqx_conf.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + -export([ load/0, admins/1, @@ -27,6 +31,7 @@ %% kept cluster_call for compatibility -define(CLUSTER_CALL, cluster_call). -define(CONF, conf). +-define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited"). load() -> emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), @@ -42,10 +47,16 @@ conf(["show"]) -> print_hocon(get_config()); conf(["show", Key]) -> print_hocon(get_config(Key)); +conf(["load", "--auth-chains", AuthChains, Path]) when + AuthChains =:= "replace"; AuthChains =:= "merge" +-> + load_config(Path, AuthChains); conf(["load", Path]) -> - load_config(Path); + load_config(Path, "replace"); conf(["cluster_sync" | Args]) -> admins(Args); +conf(["reload"]) -> + reload_etc_conf_on_local_node(); conf(_) -> emqx_ctl:usage(usage_conf() ++ usage_sync()). @@ -87,8 +98,7 @@ admins(_) -> usage_conf() -> [ - %% TODO add reload - %{"conf reload", "reload etc/emqx.conf on local node"}, + {"conf reload", "reload etc/emqx.conf on local node"}, {"conf show_keys", "Print all config keys"}, {"conf show []", "Print in-use configs (including default values) under the given key. " @@ -138,11 +148,14 @@ print_keys(Config) -> print(Json) -> emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]). -print_hocon(Hocon) -> - emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]). +print_hocon(Hocon) when is_map(Hocon) -> + emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]); +print_hocon({error, Error}) -> + emqx_ctl:warning("~ts~n", [Error]). get_config() -> - drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))). + AllConf = emqx_config:fill_defaults(emqx:get_raw_config([])), + drop_hidden_roots(AllConf). drop_hidden_roots(Conf) -> Hidden = hidden_roots(), @@ -164,22 +177,183 @@ hidden_roots() -> ). get_config(Key) -> - emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}). + case emqx:get_raw_config([Key], undefined) of + undefined -> {error, "key_not_found"}; + Value -> emqx_config:fill_defaults(#{Key => Value}) + end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -load_config(Path) -> +load_config(Path, AuthChain) -> case hocon:files([Path]) of - {ok, Conf} -> - maps:foreach( - fun(Key, Value) -> - case emqx_conf:update([Key], Value, ?OPTIONS) of - {ok, _} -> emqx_ctl:print("load ~ts ok~n", [Key]); - {error, Reason} -> emqx_ctl:print("load ~ts failed: ~p~n", [Key, Reason]) - end - end, - Conf - ); + {ok, RawConf} when RawConf =:= #{} -> + emqx_ctl:warning("load ~ts is empty~n", [Path]), + {error, empty_hocon_file}; + {ok, RawConf} -> + case check_config(RawConf) of + ok -> + lists:foreach( + fun({K, V}) -> update_config(K, V, AuthChain) end, + to_sorted_list(RawConf) + ); + {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> + emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), + emqx_ctl:warning( + "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" + ), + {error, Reason}; + {error, Errors} -> + emqx_ctl:warning("load ~ts schema check failed~n", [Path]), + lists:foreach( + fun({Key, Error}) -> + emqx_ctl:warning("~ts: ~p~n", [Key, Error]) + end, + Errors + ), + {error, Errors} + end; {error, Reason} -> - emqx_ctl:print("load ~ts failed~n~p~n", [Path, Reason]), + emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), {error, bad_hocon_file} end. + +update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> + check_res(Key, emqx_authz:merge(Conf)); +update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> + check_res(Key, emqx_authn:merge_config(Conf)); +update_config(Key, Value, _) -> + check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)). + +check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); +check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]). + +check_config(Conf) -> + case check_keys_is_not_readonly(Conf) of + ok -> check_config_schema(Conf); + Error -> Error + end. + +check_keys_is_not_readonly(Conf) -> + Keys = maps:keys(Conf), + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], + case ReadOnlyKeys -- Keys of + ReadOnlyKeys -> ok; + _ -> {error, ?UPDATE_READONLY_KEYS_PROHIBITED} + end. + +check_config_schema(Conf) -> + SchemaMod = emqx_conf:schema_module(), + Fold = fun({Key, Value}, Acc) -> + Schema = emqx_config_handler:schema(SchemaMod, [Key]), + case emqx_conf:check_config(Schema, #{Key => Value}) of + {ok, _} -> Acc; + {error, Reason} -> [{Key, Reason} | Acc] + end + end, + sorted_fold(Fold, Conf). + +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} + end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = emqx_config:config_files(), + Opts = #{format => map, include_dirs => emqx_config:include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = emqx_config:has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = emqx_conf:schema_module(), + RawDefault = emqx_config:fill_defaults(Raw), + case emqx_conf:check_config(SchemaMod, RawDefault) of + {ok, CheckedConf} -> + case filter_changed_readonly_keys(CheckedConf) of + [] -> + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], + {ok, maps:without(ReadOnlyKeys, Raw)}; + Error -> + ?SLOG(error, #{ + msg => ?UPDATE_READONLY_KEYS_PROHIBITED, + read_only_keys => ?READONLY_KEYS, + error => Error + }), + {error, Error} + end; + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_etc_config_schema_found", + error => Error + }), + {error, Error} + end. + +reload_config(AllConf) -> + Fold = fun({Key, Conf}, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + emqx_ctl:print("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + [{Key, Error} | Acc] + end + end, + sorted_fold(Fold, AllConf). + +filter_changed_readonly_keys(Conf) -> + lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS). + +filter_changed(Key, ChangedConf) -> + Prev = emqx_conf:get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + +sorted_fold(Func, Conf) -> + case lists:foldl(Func, [], to_sorted_list(Conf)) of + [] -> ok; + Error -> {error, Error} + end. + +to_sorted_list(Conf) -> + lists:keysort(1, maps:to_list(Conf)). diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl new file mode 100644 index 000000000..0e0cc0127 --- /dev/null +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -0,0 +1,131 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_conf_cli_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include("emqx_conf.hrl"). +-import(emqx_config_SUITE, [prepare_conf_file/3]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]). + +t_load_config(Config) -> + Authz = authorization, + Conf = emqx_conf:get_raw([Authz]), + %% set sources to [] + ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ok = emqx_conf_cli:conf(["load", ConfFile0]), + ?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])), + %% remove sources, it will reset to default file source. + ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + ok = emqx_conf_cli:conf(["load", ConfFile1]), + Default = [emqx_authz_schema:default_authz()], + ?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])), + %% reset + ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}), + ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config), + ok = emqx_conf_cli:conf(["load", ConfFile2]), + ?assertEqual( + Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]}, + emqx_conf:get_raw([Authz]) + ), + ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])), + ok. + +t_load_readonly(Config) -> + Base0 = base_conf(), + Base1 = Base0#{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, + lists:foreach( + fun(Key) -> + KeyBin = atom_to_binary(Key), + Conf = emqx_conf:get_raw([Key]), + ConfBin0 = hocon_pp:do(Base1#{KeyBin => Conf}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ?assertEqual( + {error, "update_readonly_keys_prohibited"}, + emqx_conf_cli:conf(["load", ConfFile0]) + ), + %% reload etc/emqx.conf changed readonly keys + ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + application:set_env(emqx, config_files, [ConfFile1]), + ?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"])) + end, + ?READONLY_KEYS + ), + ok. + +t_error_schema_check(Config) -> + Base = #{ + %% bad multiplier + <<"mqtt">> => #{<<"keepalive_multiplier">> => -1}, + <<"zones">> => #{<<"my-zone">> => #{<<"mqtt">> => #{<<"keepalive_multiplier">> => 10}}} + }, + ConfBin0 = hocon_pp:do(Base, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ?assertMatch({error, _}, emqx_conf_cli:conf(["load", ConfFile0])), + %% zones is not updated because of error + ?assertEqual(#{}, emqx_config:get_raw([zones])), + ok. + +t_reload_etc_emqx_conf_not_persistent(Config) -> + Mqtt = emqx_conf:get_raw([mqtt]), + Base = base_conf(), + Conf = Base#{<<"mqtt">> => Mqtt#{<<"keepalive_multiplier">> => 3}}, + ConfBin = hocon_pp:do(Conf, #{}), + ConfFile = prepare_conf_file(?FUNCTION_NAME, ConfBin, Config), + application:set_env(emqx, config_files, [ConfFile]), + ok = emqx_conf_cli:conf(["reload"]), + ?assertEqual(3, emqx:get_config([mqtt, keepalive_multiplier])), + ?assertNotEqual( + 3, + emqx_utils_maps:deep_get( + [<<"mqtt">>, <<"keepalive_multiplier">>], + emqx_config:read_override_conf(#{}), + undefined + ) + ), + ok. + +base_conf() -> + #{ + <<"cluster">> => emqx_conf:get_raw([cluster]), + <<"node">> => emqx_conf:get_raw([node]) + }. + +changed(cluster) -> + #{<<"name">> => <<"emqx-test">>}; +changed(node) -> + #{ + <<"name">> => <<"emqx-test@127.0.0.1">>, + <<"cookie">> => <<"gokdfkdkf1122">>, + <<"data_dir">> => <<"data">> + }; +changed(rpc) -> + #{<<"mode">> => <<"sync">>}. diff --git a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl index cc874756d..ba74ed986 100644 --- a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl @@ -62,7 +62,7 @@ end_per_suite(_Config) -> t_log_conf(_Conf) -> FileExpect = #{ <<"enable">> => true, - <<"formatter">> => text, + <<"formatter">> => <<"text">>, <<"level">> => <<"info">>, <<"rotation_count">> => 10, <<"rotation_size">> => <<"50MB">>, @@ -73,7 +73,7 @@ t_log_conf(_Conf) -> <<"console">> => #{ <<"enable">> => true, - <<"formatter">> => text, + <<"formatter">> => <<"text">>, <<"level">> => <<"debug">>, <<"time_offset">> => <<"system">> }, diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 76068d361..d1a7ed1d7 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -38,6 +38,8 @@ -export([ print/1, print/2, + warning/1, + warning/2, usage/1, usage/2 ]). @@ -180,6 +182,14 @@ print(Msg) -> print(Format, Args) -> io:format("~ts", [format(Format, Args)]). +-spec warning(io:format()) -> ok. +warning(Format) -> + warning(Format, []). + +-spec warning(io:format(), [term()]) -> ok. +warning(Format, Args) -> + io:format("\e[31m~ts\e[0m", [format(Format, Args)]). + -spec usage([cmd_usage()]) -> ok. usage(UsageList) -> io:format(format_usage(UsageList)). diff --git a/mix.exs b/mix.exs index 455f2e6a9..028a5dfff 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.7", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.8", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index d0e9570f8..57661f75f 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}