From c819ac27f4858aa01bb0f219e25632e510c0e598 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 8 Jun 2023 17:56:31 +0800 Subject: [PATCH 01/12] feat: add conf reload cli --- apps/emqx/include/emqx.hrl | 3 + apps/emqx/src/emqx_config.erl | 99 +++++++++++++++++++++++++++- apps/emqx_authz/src/emqx_authz.erl | 6 +- apps/emqx_conf/src/emqx_conf_cli.erl | 56 +++++++++++----- apps/emqx_ctl/src/emqx_ctl.erl | 10 +++ 5 files changed, 151 insertions(+), 23 deletions(-) diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 64cd4687b..75a4a7714 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -17,6 +17,9 @@ -ifndef(EMQX_HRL). -define(EMQX_HRL, true). +%% Config +-define(READ_ONLY_KEYS, [cluster, rpc, node]). + %% Shard %%-------------------------------------------------------------------- -define(COMMON_SHARD, emqx_common_shard). diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index ff241ed0e..790f6040d 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -18,6 +18,7 @@ -compile({no_auto_import, [get/0, get/1, put/2, erase/1]}). -elvis([{elvis_style, god_modules, disable}]). -include("logger.hrl"). +-include("emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ @@ -33,7 +34,8 @@ save_configs/5, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/3 + save_to_override_conf/3, + reload_etc_conf_on_local_node/0 ]). -export([merge_envs/2]). @@ -311,8 +313,7 @@ put_raw(KeyPath0, Config) -> %% Load/Update configs From/To files %%============================================================================ init_load(SchemaMod) -> - ConfFiles = application:get_env(emqx, config_files, []), - init_load(SchemaMod, ConfFiles). + init_load(SchemaMod, config_files()). %% @doc Initial load of the given config files. %% NOTE: The order of the files is significant, configs from files ordered @@ -975,3 +976,95 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) -> ok; put_config_post_change_actions(_Key, _NewValue) -> ok. + +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} + end. + +reload_config(AllConf) -> + Func = fun(Key, Conf, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + io:format("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + Acc#{Key => Error} + end + end, + Res = maps:fold(Func, #{}, AllConf), + case Res =:= #{} of + true -> ok; + false -> {error, Res} + end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = config_files(), + Opts = #{format => map, include_dirs => include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = emqx_conf:schema_module(), + {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}), + case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of + [] -> + {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)}; + Error -> + ?SLOG(error, #{ + msg => "failed_to_change_read_only_key_in_etc_config", + read_only_keys => ?READ_ONLY_KEYS, + error => Error + }), + {error, Error} + end. + +filter_changed(Key, ChangedConf) -> + Prev = get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + +config_files() -> + application:get_env(emqx, config_files, []). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 3c9698de0..9dd46c0c2 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -166,10 +166,10 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - #{<<"sources">> := NewSources} = NewConf, - #{<<"sources">> := OldSources} = OldConf, + NewSources = maps:get(<<"sources">>, NewConf, []), + OldSources = maps:get(<<"sources">>, OldConf, []), NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), - NewConf#{<<"sources">> := NewSources1}. + NewConf#{<<"sources">> => NewSources1}. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 530e4bfcb..8d74c4cd9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_conf_cli). +-include_lib("emqx/include/emqx.hrl"). -export([ load/0, admins/1, @@ -87,8 +88,7 @@ admins(_) -> usage_conf() -> [ - %% TODO add reload - %{"conf reload", "reload etc/emqx.conf on local node"}, + {"conf reload", "reload etc/emqx.conf on local node"}, {"conf show_keys", "Print all config keys"}, {"conf show []", "Print in-use configs (including default values) under the given key. " @@ -138,11 +138,14 @@ print_keys(Config) -> print(Json) -> emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]). -print_hocon(Hocon) -> - emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]). +print_hocon(Hocon) when is_map(Hocon) -> + emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]); +print_hocon({error, Error}) -> + emqx_ctl:warning("~ts~n", [Error]). get_config() -> - drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))). + AllConf = emqx_config:fill_defaults(emqx:get_raw_config([])), + drop_hidden_roots(AllConf). drop_hidden_roots(Conf) -> Hidden = hidden_roots(), @@ -164,22 +167,41 @@ hidden_roots() -> ). get_config(Key) -> - emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}). + case emqx:get_raw_config(Key, undefined) of + undefined -> {error, "key_not_found"}; + Value -> emqx_config:fill_defaults(#{Key => Value}) + end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). load_config(Path) -> case hocon:files([Path]) of - {ok, Conf} -> - maps:foreach( - fun(Key, Value) -> - case emqx_conf:update([Key], Value, ?OPTIONS) of - {ok, _} -> emqx_ctl:print("load ~ts ok~n", [Key]); - {error, Reason} -> emqx_ctl:print("load ~ts failed: ~p~n", [Key, Reason]) - end - end, - Conf - ); + {ok, RawConf} -> + case check_config_keys(RawConf) of + ok -> + maps:foreach(fun update_config/2, RawConf); + {error, Reason} -> + emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), + emqx_ctl:warning( + "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" + ), + {error, Reason} + end; {error, Reason} -> - emqx_ctl:print("load ~ts failed~n~p~n", [Path, Reason]), + emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), {error, bad_hocon_file} end. + +update_config(Key, Value) -> + case emqx_conf:update([Key], Value, ?OPTIONS) of + {ok, _} -> + emqx_ctl:print("load ~ts in cluster ok~n", [Key]); + {error, Reason} -> + emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]) + end. +check_config_keys(Conf) -> + Keys = maps:keys(Conf), + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS], + case ReadOnlyKeys -- Keys of + ReadOnlyKeys -> ok; + _ -> {error, "update_read_only_keys_prohibited"} + end. diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 76068d361..d1a7ed1d7 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -38,6 +38,8 @@ -export([ print/1, print/2, + warning/1, + warning/2, usage/1, usage/2 ]). @@ -180,6 +182,14 @@ print(Msg) -> print(Format, Args) -> io:format("~ts", [format(Format, Args)]). +-spec warning(io:format()) -> ok. +warning(Format) -> + warning(Format, []). + +-spec warning(io:format(), [term()]) -> ok. +warning(Format, Args) -> + io:format("\e[31m~ts\e[0m", [format(Format, Args)]). + -spec usage([cmd_usage()]) -> ok. usage(UsageList) -> io:format(format_usage(UsageList)). From 994db58cbf7735465b73a6c654666108deb6a184 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 8 Jun 2023 23:55:11 +0800 Subject: [PATCH 02/12] fix: load authz's default sources --- apps/emqx/include/emqx.hrl | 3 - apps/emqx/src/emqx_config.erl | 93 +-------------------- apps/emqx_authz/src/emqx_authz.erl | 5 +- apps/emqx_authz/src/emqx_authz_schema.erl | 3 +- apps/emqx_conf/include/emqx_conf.hrl | 2 + apps/emqx_conf/src/emqx_conf.erl | 92 ++++++++++++++++++++ apps/emqx_conf/src/emqx_conf_cli.erl | 11 ++- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 72 ++++++++++++++++ 8 files changed, 182 insertions(+), 99 deletions(-) create mode 100644 apps/emqx_conf/test/emqx_conf_cli_SUITE.erl diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 75a4a7714..64cd4687b 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -17,9 +17,6 @@ -ifndef(EMQX_HRL). -define(EMQX_HRL, true). -%% Config --define(READ_ONLY_KEYS, [cluster, rpc, node]). - %% Shard %%-------------------------------------------------------------------- -define(COMMON_SHARD, emqx_common_shard). diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 790f6040d..2f7a4350a 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -35,7 +35,8 @@ save_to_app_env/1, save_to_config_map/2, save_to_override_conf/3, - reload_etc_conf_on_local_node/0 + config_files/0, + include_dirs/0 ]). -export([merge_envs/2]). @@ -91,6 +92,7 @@ ]). -export([ensure_atom_conf_path/2]). +-export([load_config_files/2]). -ifdef(TEST). -export([erase_all/0, backup_and_write/2]). @@ -977,94 +979,5 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) -> put_config_post_change_actions(_Key, _NewValue) -> ok. -%% @doc Reload etc/emqx.conf to runtime config except for the readonly config --spec reload_etc_conf_on_local_node() -> ok | {error, term()}. -reload_etc_conf_on_local_node() -> - case load_etc_config_file() of - {ok, RawConf} -> - case check_readonly_config(RawConf) of - {ok, Reloaded} -> reload_config(Reloaded); - {error, Error} -> {error, Error} - end; - {error, _Error} -> - {error, bad_hocon_file} - end. - -reload_config(AllConf) -> - Func = fun(Key, Conf, Acc) -> - case emqx:update_config([Key], Conf, #{persistent => false}) of - {ok, _} -> - io:format("Reloaded ~ts config ok~n", [Key]), - Acc; - Error -> - ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), - ?SLOG(error, #{ - msg => "failed_to_reload_etc_config", - key => Key, - value => Conf, - error => Error - }), - Acc#{Key => Error} - end - end, - Res = maps:fold(Func, #{}, AllConf), - case Res =:= #{} of - true -> ok; - false -> {error, Res} - end. - -%% @doc Merge etc/emqx.conf on top of cluster.hocon. -%% For example: -%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, -%% but `authorization.sources` in not in the default emqx.conf file. -%% To make sure all root keys in emqx.conf has a fully merged value. -load_etc_config_file() -> - ConfFiles = config_files(), - Opts = #{format => map, include_dirs => include_dirs()}, - case hocon:files(ConfFiles, Opts) of - {ok, RawConf} -> - HasDeprecatedFile = has_deprecated_file(), - %% Merge etc.conf on top of cluster.hocon, - %% Don't use map deep_merge, use hocon files merge instead. - %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) - Keys = maps:keys(RawConf), - MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles), - {ok, maps:with(Keys, MergedRaw)}; - {error, Error} -> - ?SLOG(error, #{ - msg => "failed_to_read_etc_config", - files => ConfFiles, - error => Error - }), - {error, Error} - end. - -check_readonly_config(Raw) -> - SchemaMod = emqx_conf:schema_module(), - {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}), - case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of - [] -> - {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)}; - Error -> - ?SLOG(error, #{ - msg => "failed_to_change_read_only_key_in_etc_config", - read_only_keys => ?READ_ONLY_KEYS, - error => Error - }), - {error, Error} - end. - -filter_changed(Key, ChangedConf) -> - Prev = get([Key], #{}), - New = maps:get(Key, ChangedConf, #{}), - case Prev =/= New of - true -> {true, {Key, changed(New, Prev)}}; - false -> false - end. - -changed(New, Prev) -> - Diff = emqx_utils_maps:diff_maps(New, Prev), - maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). - config_files() -> application:get_env(emqx, config_files, []). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 9dd46c0c2..b4c943072 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -166,8 +166,9 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - NewSources = maps:get(<<"sources">>, NewConf, []), - OldSources = maps:get(<<"sources">>, OldConf, []), + Default = [emqx_authz_schema:default_authz()], + NewSources = maps:get(<<"sources">>, NewConf, Default), + OldSources = maps:get(<<"sources">>, OldConf, Default), NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), NewConf#{<<"sources">> => NewSources1}. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 8e847b93e..b19c62441 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,7 +42,8 @@ -export([ headers_no_content_type/1, - headers/1 + headers/1, + default_authz/0 ]). %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index 0297fddf7..26042d62c 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -34,4 +34,6 @@ tnx_id :: pos_integer() | '$1' }). +-define(READONLY_KEYS, [cluster, rpc, node]). + -endif. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d2e45581d..b9e5e17fa 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -19,6 +19,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/emqx_schema.hrl"). +-include("emqx_conf.hrl"). -export([add_handler/2, remove_handler/1]). -export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]). @@ -30,6 +31,7 @@ -export([dump_schema/2]). -export([schema_module/0]). -export([gen_example_conf/2]). +-export([reload_etc_conf_on_local_node/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ @@ -213,10 +215,100 @@ schema_module() -> Value -> list_to_existing_atom(Value) end. +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} + end. + +reload_config(AllConf) -> + Func = fun(Key, Conf, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + io:format("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + Acc#{Key => Error} + end + end, + Res = maps:fold(Func, #{}, AllConf), + case Res =:= #{} of + true -> ok; + false -> {error, Res} + end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = emqx_config:config_files(), + Opts = #{format => map, include_dirs => emqx_config:include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = emqx_config:has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = schema_module(), + RawDefault = emqx_config:fill_defaults(Raw), + {_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault), + case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of + [] -> + {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)}; + Error -> + ?SLOG(error, #{ + msg => "failed_to_change_read_only_key_in_etc_config", + read_only_keys => ?READONLY_KEYS, + error => Error + }), + {error, Error} + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +filter_changed(Key, ChangedConf) -> + Prev = get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. make_desc_resolver(Lang) -> fun diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 8d74c4cd9..eb8057fc9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_conf_cli). --include_lib("emqx/include/emqx.hrl"). +-include("emqx_conf.hrl"). -export([ load/0, admins/1, @@ -47,6 +47,8 @@ conf(["load", Path]) -> load_config(Path); conf(["cluster_sync" | Args]) -> admins(Args); +conf(["reload"]) -> + emqx_conf:reload_etc_conf_on_local_node(); conf(_) -> emqx_ctl:usage(usage_conf() ++ usage_sync()). @@ -175,6 +177,9 @@ get_config(Key) -> -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). load_config(Path) -> case hocon:files([Path]) of + {ok, RawConf} when RawConf =:= #{} -> + emqx_ctl:warning("load ~ts is empty~n", [Path]), + {error, empty_hocon_file}; {ok, RawConf} -> case check_config_keys(RawConf) of ok -> @@ -200,8 +205,8 @@ update_config(Key, Value) -> end. check_config_keys(Conf) -> Keys = maps:keys(Conf), - ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS], + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case ReadOnlyKeys -- Keys of ReadOnlyKeys -> ok; - _ -> {error, "update_read_only_keys_prohibited"} + _ -> {error, "update_readonly_keys_prohibited"} end. diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl new file mode 100644 index 000000000..3ca90646b --- /dev/null +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_conf_cli_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include("emqx_conf.hrl"). +-import(emqx_config_SUITE, [prepare_conf_file/3]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]). + +t_load_config(Config) -> + Authz = authorization, + Conf = emqx_conf:get_raw([Authz]), + %% set sources to [] + ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ok = emqx_conf_cli:conf(["load", ConfFile0]), + ?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])), + %% remove sources, it will reset to default file source. + ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + ok = emqx_conf_cli:conf(["load", ConfFile1]), + Default = [emqx_authz_schema:default_authz()], + ?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])), + %% reset + ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}), + ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config), + ok = emqx_conf_cli:conf(["load", ConfFile2]), + ?assertEqual(Conf, emqx_conf:get_raw([Authz])), + ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])), + ok. + +t_load_readonly(Config) -> + Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, + lists:foreach( + fun(Key) -> + Conf = emqx_conf:get_raw([Key]), + ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ?assertEqual( + {error, "update_readonly_keys_prohibited"}, + emqx_conf_cli:conf(["load", ConfFile0]) + ) + end, + ?READONLY_KEYS + ), + ok. From aac5b85d267b68fd4a71cb245925deafb948e4a1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Jun 2023 11:16:40 +0800 Subject: [PATCH 03/12] feat: authz support merge sources --- apps/emqx_authz/include/emqx_authz.hrl | 1 + apps/emqx_authz/src/emqx_authz.erl | 91 ++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 967865868..b43a2cdab 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -37,6 +37,7 @@ -define(CMD_PREPEND, prepend). -define(CMD_APPEND, append). -define(CMD_MOVE, move). +-define(CMD_MERGE, merge). -define(CMD_MOVE_FRONT, front). -define(CMD_MOVE_REAR, rear). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index b4c943072..7deda2726 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -24,11 +24,6 @@ -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - -export([ register_metrics/0, init/0, @@ -37,6 +32,7 @@ lookup/1, move/2, update/2, + merge/1, authorize/5, %% for telemetry information get_enabled_authzs/0 @@ -45,6 +41,7 @@ -export([post_config_update/5, pre_config_update/3]). -export([acl_conf_file/0]). +-export([merge_sources/2, search/2]). %% Data backup -export([ @@ -128,6 +125,9 @@ lookup(Type) -> {Source, _Front, _Rear} = take(Type), Source. +merge(NewConf) -> + emqx_authz_utils:update_config(?ROOT_KEY, {?CMD_MERGE, NewConf}). + move(Type, ?CMD_MOVE_BEFORE(Before)) -> emqx_authz_utils:update_config( ?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))} @@ -158,9 +158,16 @@ pre_config_update(Path, Cmd, Sources) -> do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources) -> do_pre_config_update(Cmd, Sources); +do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf) -> + do_pre_config_merge(NewConf, OldConf); do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(NewConf, OldConf). +do_pre_config_merge(NewConf, OldConf) -> + MergeConf0 = emqx_utils_maps:deep_merge(OldConf, NewConf), + NewSources = merge_sources(NewConf, OldConf), + do_pre_config_replace(MergeConf0#{<<"sources">> := NewSources}, OldConf). + %% override the entire config when updating the root key %% emqx_conf:update(?ROOT_KEY, Conf); do_pre_config_replace(Conf, Conf) -> @@ -629,3 +636,77 @@ check_acl_file_rules(Path, Rules) -> after _ = file:delete(TmpPath) end. + +merge_sources(OldConf, NewConf) -> + Default = [emqx_authz_schema:default_authz()], + NewSources0 = maps:get(<<"sources">>, NewConf, Default), + OriginSources0 = maps:get(<<"sources">>, OldConf, Default), + {OriginSource, NewSources} = + lists:foldl( + fun(Old = #{<<"type">> := Type}, {OriginAcc, NewAcc}) -> + case search(Type, NewAcc) of + not_found -> + {[Old | OriginAcc], NewAcc}; + {New, NewAcc1} -> + MergeSource = emqx_utils_maps:deep_merge(Old, New), + {[MergeSource | OriginAcc], NewAcc1} + end + end, + {[], NewSources0}, + OriginSources0 + ), + lists:reverse(OriginSource) ++ NewSources. + +search(Type, Sources) -> + case lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources) of + {_Front, []} -> not_found; + {Front, [Target | Rear]} -> {Target, Front ++ Rear} + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-compile(nowarn_export_all). +-compile(export_all). + +merge_sources_test() -> + Default = [emqx_authz_schema:default_authz()], + Http = #{<<"type">> => <<"http">>, <<"enable">> => true}, + Mysql = #{<<"type">> => <<"mysql">>, <<"enable">> => true}, + Mongo = #{<<"type">> => <<"mongodb">>, <<"enable">> => true}, + Redis = #{<<"type">> => <<"redis">>, <<"enable">> => true}, + HttpDisable = Http#{<<"enable">> => false}, + MysqlDisable = Mysql#{<<"enable">> => false}, + MongoDisable = Mongo#{<<"enable">> => false}, + + %% has default source + ?assertEqual(Default, merge_sources(#{}, #{})), + ?assertEqual([], merge_sources(#{<<"sources">> => []}, #{<<"sources">> => []})), + ?assertEqual(Default, merge_sources(#{}, #{<<"sources">> => []})), + + %% add + ?assertEqual( + [Http, Mysql, Mongo, Redis], + merge_sources( + #{<<"sources">> => [Http, Mysql]}, + #{<<"sources">> => [Mongo, Redis]} + ) + ), + %% replace + ?assertEqual( + [HttpDisable, MysqlDisable], + merge_sources( + #{<<"sources">> => [Http, Mysql]}, + #{<<"sources">> => [HttpDisable, MysqlDisable]} + ) + ), + %% add + replace + change position + ?assertEqual( + [HttpDisable, Mysql, MongoDisable, Redis], + merge_sources( + #{<<"sources">> => [Http, Mysql, Mongo]}, + #{<<"sources">> => [MongoDisable, HttpDisable, Redis]} + ) + ), + ok. + +-endif. From 21ffd958b4da30a9b70245e4f052c49c47c8db93 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Jun 2023 12:04:14 +0800 Subject: [PATCH 04/12] feat: support authn merge_authenticators --- apps/emqx/include/emqx_authentication.hrl | 1 + apps/emqx/src/emqx_authentication_config.erl | 81 +++++++++++++++++++- apps/emqx_authn/src/emqx_authn.erl | 8 +- apps/emqx_authz/src/emqx_authz.erl | 5 +- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl index 20ae2bf1e..70b35a474 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx/include/emqx_authentication.hrl @@ -47,5 +47,6 @@ -define(CMD_MOVE_REAR, rear). -define(CMD_MOVE_BEFORE(Before), {before, Before}). -define(CMD_MOVE_AFTER(After), {'after', After}). +-define(CMD_MERGE, merge). -endif. diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 92041095b..96718d611 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -55,7 +55,9 @@ {create_authenticator, chain_name(), map()} | {delete_authenticator, chain_name(), authenticator_id()} | {update_authenticator, chain_name(), authenticator_id(), map()} - | {move_authenticator, chain_name(), authenticator_id(), position()}. + | {move_authenticator, chain_name(), authenticator_id(), position()} + | {merge_authenticators, map()} + | map(). %%------------------------------------------------------------------------------ %% Callbacks of config handler @@ -128,6 +130,9 @@ do_pre_config_update(_, {move_authenticator, _ChainName, AuthenticatorID, Positi end end end; +do_pre_config_update(Paths, {merge_authenticators, NewConfig}, OldConfig) -> + MergeConfig = merge_authenticators(OldConfig, NewConfig), + do_pre_config_update(Paths, MergeConfig, OldConfig); do_pre_config_update(_, OldConfig, OldConfig) -> {ok, OldConfig}; do_pre_config_update(Paths, NewConfig, _OldConfig) -> @@ -327,3 +332,77 @@ chain_name([authentication]) -> ?GLOBAL; chain_name([listeners, Type, Name, authentication]) -> binary_to_existing_atom(<<(atom_to_binary(Type))/binary, ":", (atom_to_binary(Name))/binary>>). + +merge_authenticators(OriginConf0, NewConf0) -> + {OriginConf1, NewConf1} = + lists:foldl( + fun(Origin, {OriginAcc, NewAcc}) -> + AuthenticatorID = authenticator_id(Origin), + case split_by_id(AuthenticatorID, NewAcc) of + {error, _} -> + {[Origin | OriginAcc], NewAcc}; + {ok, BeforeFound, [Found | AfterFound]} -> + Merged = emqx_utils_maps:deep_merge(Origin, Found), + {[Merged | OriginAcc], BeforeFound ++ AfterFound} + end + end, + {[], NewConf0}, + OriginConf0 + ), + lists:reverse(OriginConf1) ++ NewConf1. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-compile(nowarn_export_all). +-compile(export_all). + +merge_authenticators_test() -> + ?assertEqual([], merge_authenticators([], [])), + + Http = #{ + <<"mechanism">> => <<"password_based">>, <<"backend">> => <<"http">>, <<"enable">> => true + }, + Jwt = #{<<"mechanism">> => <<"jwt">>, <<"enable">> => true}, + BuildIn = #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"built_in_database">>, + <<"enable">> => true + }, + Mongodb = #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"mongodb">>, + <<"enable">> => true + }, + Redis = #{ + <<"mechanism">> => <<"password_based">>, <<"backend">> => <<"redis">>, <<"enable">> => true + }, + BuildInDisable = BuildIn#{<<"enable">> => false}, + MongodbDisable = Mongodb#{<<"enable">> => false}, + RedisDisable = Redis#{<<"enable">> => false}, + + %% add + ?assertEqual([Http], merge_authenticators([], [Http])), + ?assertEqual([Http, Jwt, BuildIn], merge_authenticators([Http], [Jwt, BuildIn])), + + %% merge + ?assertEqual( + [BuildInDisable, MongodbDisable], + merge_authenticators([BuildIn, Mongodb], [BuildInDisable, MongodbDisable]) + ), + ?assertEqual( + [BuildInDisable, Jwt], + merge_authenticators([BuildIn, Jwt], [BuildInDisable]) + ), + ?assertEqual( + [BuildInDisable, Jwt, Mongodb], + merge_authenticators([BuildIn, Jwt], [Mongodb, BuildInDisable]) + ), + + %% position changed + ?assertEqual( + [BuildInDisable, Jwt, Mongodb, RedisDisable, Http], + merge_authenticators([BuildIn, Jwt, Mongodb, Redis], [RedisDisable, BuildInDisable, Http]) + ), + ok. + +-endif. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 2a8d82439..e413a409a 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -26,10 +26,7 @@ get_enabled_authns/0 ]). -%% Data backup --export([ - import_config/1 -]). +-export([merge_config/1, import_config/1]). -include("emqx_authn.hrl"). @@ -162,3 +159,6 @@ authn_list(Authn) when is_list(Authn) -> Authn; authn_list(Authn) when is_map(Authn) -> [Authn]. + +merge_config(AuthNs) -> + emqx_authn_api:update_config(?CONF_NS_BINARY, {merge_authenticators, AuthNs}). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 7deda2726..eb8fbefba 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -674,6 +674,7 @@ merge_sources_test() -> Mysql = #{<<"type">> => <<"mysql">>, <<"enable">> => true}, Mongo = #{<<"type">> => <<"mongodb">>, <<"enable">> => true}, Redis = #{<<"type">> => <<"redis">>, <<"enable">> => true}, + Postgresql = #{<<"type">> => <<"postgresql">>, <<"enable">> => true}, HttpDisable = Http#{<<"enable">> => false}, MysqlDisable = Mysql#{<<"enable">> => false}, MongoDisable = Mongo#{<<"enable">> => false}, @@ -685,10 +686,10 @@ merge_sources_test() -> %% add ?assertEqual( - [Http, Mysql, Mongo, Redis], + [Http, Mysql, Mongo, Redis, Postgresql], merge_sources( #{<<"sources">> => [Http, Mysql]}, - #{<<"sources">> => [Mongo, Redis]} + #{<<"sources">> => [Mongo, Redis, Postgresql]} ) ), %% replace From b829d8edc2a42fe0e82030be4c193f00fb9aedd5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Jun 2023 12:55:28 +0800 Subject: [PATCH 05/12] feat: support --auth-chains merge|replace mod --- apps/emqx_authz/src/emqx_authz.erl | 1 - apps/emqx_conf/src/emqx_conf_cli.erl | 35 +++++++++++++++++++--------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index eb8fbefba..0109a08b4 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -41,7 +41,6 @@ -export([post_config_update/5, pre_config_update/3]). -export([acl_conf_file/0]). --export([merge_sources/2, search/2]). %% Data backup -export([ diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index eb8057fc9..84aef48f2 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -16,6 +16,9 @@ -module(emqx_conf_cli). -include("emqx_conf.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + -export([ load/0, admins/1, @@ -43,8 +46,12 @@ conf(["show"]) -> print_hocon(get_config()); conf(["show", Key]) -> print_hocon(get_config(Key)); +conf(["load", "--auth-chains", AuthChains, Path]) when + AuthChains =:= "replace"; AuthChains =:= "merge" +-> + load_config(Path, AuthChains); conf(["load", Path]) -> - load_config(Path); + load_config(Path, "replace"); conf(["cluster_sync" | Args]) -> admins(Args); conf(["reload"]) -> @@ -169,13 +176,13 @@ hidden_roots() -> ). get_config(Key) -> - case emqx:get_raw_config(Key, undefined) of + case emqx:get_raw_config([Key], undefined) of undefined -> {error, "key_not_found"}; Value -> emqx_config:fill_defaults(#{Key => Value}) end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -load_config(Path) -> +load_config(Path, AuthChain) -> case hocon:files([Path]) of {ok, RawConf} when RawConf =:= #{} -> emqx_ctl:warning("load ~ts is empty~n", [Path]), @@ -183,7 +190,7 @@ load_config(Path) -> {ok, RawConf} -> case check_config_keys(RawConf) of ok -> - maps:foreach(fun update_config/2, RawConf); + maps:foreach(fun(K, V) -> update_config(K, V, AuthChain) end, RawConf); {error, Reason} -> emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), emqx_ctl:warning( @@ -196,13 +203,19 @@ load_config(Path) -> {error, bad_hocon_file} end. -update_config(Key, Value) -> - case emqx_conf:update([Key], Value, ?OPTIONS) of - {ok, _} -> - emqx_ctl:print("load ~ts in cluster ok~n", [Key]); - {error, Reason} -> - emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]) - end. +update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> + Res = emqx_authz:merge(Conf), + check_res(Key, Res); +update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> + Res = emqx_authn:merge_config(Conf), + check_res(Key, Res); +update_config(Key, Value, _) -> + Res = emqx_conf:update([Key], Value, ?OPTIONS), + check_res(Key, Res). + +check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); +check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]). + check_config_keys(Conf) -> Keys = maps:keys(Conf), ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], From cf9a207743faaca639fe05ed3250d3e8ee5cf457 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Jun 2023 15:28:28 +0800 Subject: [PATCH 06/12] feat: check all config schema before loading --- apps/emqx/src/emqx_config_handler.erl | 2 + apps/emqx_authn/src/emqx_authn.erl | 2 +- apps/emqx_authn/src/emqx_authn_api.erl | 2 + apps/emqx_conf/src/emqx_conf.erl | 34 ++++++++--- apps/emqx_conf/src/emqx_conf_cli.erl | 34 ++++++++++- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 62 ++++++++++++++++++++- 6 files changed, 122 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 1ca2e8b24..0b5cc695d 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -44,6 +44,8 @@ code_change/3 ]). +-export([schema/2]). + -define(MOD, {mod}). -define(WKEY, '?'). diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index e413a409a..e717550f1 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -161,4 +161,4 @@ authn_list(Authn) when is_map(Authn) -> [Authn]. merge_config(AuthNs) -> - emqx_authn_api:update_config(?CONF_NS_BINARY, {merge_authenticators, AuthNs}). + emqx_authn_api:update_config([?CONF_NS_ATOM], {merge_authenticators, AuthNs}). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 65ce0cc32..b237108c2 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -89,6 +89,8 @@ param_listener_id/0 ]). +-export([update_config/2]). + -elvis([{elvis_style, god_modules, disable}]). api_spec() -> diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index b9e5e17fa..483241c37 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -32,6 +32,7 @@ -export([schema_module/0]). -export([gen_example_conf/2]). -export([reload_etc_conf_on_local_node/0]). +-export([check_config/2]). %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ @@ -280,19 +281,38 @@ load_etc_config_file() -> check_readonly_config(Raw) -> SchemaMod = schema_module(), RawDefault = emqx_config:fill_defaults(Raw), - {_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault), - case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of - [] -> - {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)}; - Error -> + case check_config(SchemaMod, RawDefault) of + {ok, CheckedConf} -> + case + lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) + of + [] -> + {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)}; + Error -> + ?SLOG(error, #{ + msg => "failed_to_change_read_only_key_in_etc_config", + read_only_keys => ?READONLY_KEYS, + error => Error + }), + {error, Error} + end; + {error, Error} -> ?SLOG(error, #{ - msg => "failed_to_change_read_only_key_in_etc_config", - read_only_keys => ?READONLY_KEYS, + msg => "failed_to_check_etc_config", error => Error }), {error, Error} end. +check_config(Mod, Raw) -> + try + {_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw), + {ok, CheckedConf} + catch + throw:Error -> + {error, Error} + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 84aef48f2..4aae4040b 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -188,14 +188,17 @@ load_config(Path, AuthChain) -> emqx_ctl:warning("load ~ts is empty~n", [Path]), {error, empty_hocon_file}; {ok, RawConf} -> - case check_config_keys(RawConf) of + case check_config(RawConf) of ok -> maps:foreach(fun(K, V) -> update_config(K, V, AuthChain) end, RawConf); - {error, Reason} -> + {error, Reason} when is_list(Reason) -> emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), emqx_ctl:warning( "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" ), + {error, Reason}; + {error, Reason} when is_map(Reason) -> + emqx_ctl:warning("load ~ts schema check failed~n~p~n", [Path, Reason]), {error, Reason} end; {error, Reason} -> @@ -216,10 +219,35 @@ update_config(Key, Value, _) -> check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]). -check_config_keys(Conf) -> +check_config(Conf) -> + case check_keys_is_not_readonly(Conf) of + ok -> check_config_schema(Conf); + Error -> Error + end. + +check_keys_is_not_readonly(Conf) -> Keys = maps:keys(Conf), ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case ReadOnlyKeys -- Keys of ReadOnlyKeys -> ok; _ -> {error, "update_readonly_keys_prohibited"} end. + +check_config_schema(Conf) -> + SchemaMod = emqx_conf:schema_module(), + Res = + maps:fold( + fun(Key, Value, Acc) -> + Schema = emqx_config_handler:schema(SchemaMod, [Key]), + case emqx_conf:check_config(Schema, #{Key => Value}) of + {ok, _} -> Acc; + {error, Reason} -> #{Key => Reason} + end + end, + #{}, + Conf + ), + case Res =:= #{} of + true -> ok; + false -> {error, Res} + end. diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl index 3ca90646b..a485f4edd 100644 --- a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -56,17 +56,73 @@ t_load_config(Config) -> ok. t_load_readonly(Config) -> - Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, + Base0 = base_conf(), + Base1 = Base0#{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, lists:foreach( fun(Key) -> + KeyBin = atom_to_binary(Key), Conf = emqx_conf:get_raw([Key]), - ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}), + ConfBin0 = hocon_pp:do(Base1#{KeyBin => Conf}, #{}), ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), ?assertEqual( {error, "update_readonly_keys_prohibited"}, emqx_conf_cli:conf(["load", ConfFile0]) - ) + ), + %% reload etc/emqx.conf changed readonly keys + ConfBin1 = hocon_pp:do(Base1#{KeyBin => changed(Key)}, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + application:set_env(emqx, config_files, [ConfFile1]), + ?assertMatch({error, [{Key, #{changed := _}}]}, emqx_conf_cli:conf(["reload"])) end, ?READONLY_KEYS ), ok. + +t_error_schema_check(Config) -> + Base = #{ + %% bad multiplier + <<"mqtt">> => #{<<"keepalive_multiplier">> => -1}, + <<"zones">> => #{<<"my-zone">> => #{<<"mqtt">> => #{<<"keepalive_multiplier">> => 10}}} + }, + ConfBin0 = hocon_pp:do(Base, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ?assertMatch({error, _}, emqx_conf_cli:conf(["load", ConfFile0])), + %% zones is not updated because of error + ?assertEqual(#{}, emqx_config:get_raw([zones])), + ok. + +t_reload_etc_emqx_conf_not_persistent(Config) -> + Mqtt = emqx_conf:get_raw([mqtt]), + Base = base_conf(), + Conf = Base#{<<"mqtt">> => Mqtt#{<<"keepalive_multiplier">> => 3}}, + ConfBin = hocon_pp:do(Conf, #{}), + ConfFile = prepare_conf_file(?FUNCTION_NAME, ConfBin, Config), + application:set_env(emqx, config_files, [ConfFile]), + ok = emqx_conf_cli:conf(["reload"]), + ?assertEqual(3, emqx:get_config([mqtt, keepalive_multiplier])), + ?assertNotEqual( + 3, + emqx_utils_maps:deep_get( + [<<"mqtt">>, <<"keepalive_multiplier">>], + emqx_config:read_override_conf(#{}), + undefined + ) + ), + ok. + +base_conf() -> + #{ + <<"cluster">> => emqx_conf:get_raw([cluster]), + <<"node">> => emqx_conf:get_raw([node]) + }. + +changed(cluster) -> + #{<<"name">> => <<"emqx-test">>}; +changed(node) -> + #{ + <<"name">> => <<"emqx-test@127.0.0.1">>, + <<"cookie">> => <<"gokdfkdkf1122">>, + <<"data_dir">> => <<"data">> + }; +changed(rpc) -> + #{<<"mode">> => <<"sync">>}. From 6c6ff4ea1fa25e6bd63ee47bacf5fb9855b6d5b3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 09:53:01 +0800 Subject: [PATCH 07/12] chore: rename {mod} to module --- apps/emqx/src/emqx_config_handler.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 0b5cc695d..6948b41e5 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -46,7 +46,7 @@ -export([schema/2]). --define(MOD, {mod}). +-define(MOD, module). -define(WKEY, '?'). -type handler_name() :: module(). From 01671d82c747c3186f5437b3f2c2ad44ea5878a3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 09:54:15 +0800 Subject: [PATCH 08/12] feat: delete authorization form etc/emqx.conf --- apps/emqx_authz/etc/emqx_authz.conf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 167b12b3f..8b1378917 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,5 +1 @@ -authorization { - deny_action = ignore - no_match = allow - cache = { enable = true } -} + From 1ca1ba9e7acf380c201a9e220b5d118cc0305d69 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 10:26:43 +0800 Subject: [PATCH 09/12] fix: get_sources return default authz when sources key not found --- apps/emqx_authz/src/emqx_authz.erl | 53 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 0109a08b4..9fd6063e7 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -163,20 +163,19 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(NewConf, OldConf). do_pre_config_merge(NewConf, OldConf) -> - MergeConf0 = emqx_utils_maps:deep_merge(OldConf, NewConf), - NewSources = merge_sources(NewConf, OldConf), - do_pre_config_replace(MergeConf0#{<<"sources">> := NewSources}, OldConf). + MergeConf = emqx_utils_maps:deep_merge(OldConf, NewConf), + NewSources = merge_sources(OldConf, NewConf), + do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf). %% override the entire config when updating the root key %% emqx_conf:update(?ROOT_KEY, Conf); do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - Default = [emqx_authz_schema:default_authz()], - NewSources = maps:get(<<"sources">>, NewConf, Default), - OldSources = maps:get(<<"sources">>, OldConf, Default), - NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), - NewConf#{<<"sources">> => NewSources1}. + NewSources = get_sources(NewConf), + OldSources = get_sources(OldConf), + ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), + NewConf#{<<"sources">> => ReplaceSources}. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); @@ -472,8 +471,8 @@ get_enabled_authzs() -> %%------------------------------------------------------------------------------ import_config(#{?CONF_NS_BINARY := AuthzConf}) -> - Sources = maps:get(<<"sources">>, AuthzConf, []), - OldSources = emqx:get_raw_config(?CONF_KEY_PATH, []), + Sources = get_sources(AuthzConf), + OldSources = emqx:get_raw_config(?CONF_KEY_PATH, [emqx_authz_schema:default_authz()]), MergedSources = emqx_utils:merge_lists(OldSources, Sources, fun type/1), MergedAuthzConf = AuthzConf#{<<"sources">> => MergedSources}, case emqx_conf:update([?CONF_NS_ATOM], MergedAuthzConf, #{override_to => cluster}) of @@ -533,12 +532,12 @@ take(Type) -> take(Type, lookup()). %% Take the source of give type, the sources list is split into two parts %% front part and rear part. take(Type, Sources) -> - {Front, Rear} = lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources), - case Rear =:= [] of - true -> + Expect = type(Type), + case lists:splitwith(fun(T) -> type(T) =/= Expect end, Sources) of + {_Front, []} -> throw({not_found_source, Type}); - _ -> - {hd(Rear), Front, tl(Rear)} + {Front, [Found | Rear]} -> + {Found, Front, Rear} end. find_action_in_hooks() -> @@ -636,14 +635,11 @@ check_acl_file_rules(Path, Rules) -> _ = file:delete(TmpPath) end. -merge_sources(OldConf, NewConf) -> - Default = [emqx_authz_schema:default_authz()], - NewSources0 = maps:get(<<"sources">>, NewConf, Default), - OriginSources0 = maps:get(<<"sources">>, OldConf, Default), +merge_sources(OriginConf, NewConf) -> {OriginSource, NewSources} = lists:foldl( fun(Old = #{<<"type">> := Type}, {OriginAcc, NewAcc}) -> - case search(Type, NewAcc) of + case type_take(Type, NewAcc) of not_found -> {[Old | OriginAcc], NewAcc}; {New, NewAcc1} -> @@ -651,15 +647,20 @@ merge_sources(OldConf, NewConf) -> {[MergeSource | OriginAcc], NewAcc1} end end, - {[], NewSources0}, - OriginSources0 + {[], get_sources(NewConf)}, + get_sources(OriginConf) ), lists:reverse(OriginSource) ++ NewSources. -search(Type, Sources) -> - case lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources) of - {_Front, []} -> not_found; - {Front, [Target | Rear]} -> {Target, Front ++ Rear} +get_sources(Conf) -> + Default = [emqx_authz_schema:default_authz()], + maps:get(<<"sources">>, Conf, Default). + +type_take(Type, Sources) -> + try take(Type, Sources) of + {Found, Front, Rear} -> {Found, Front ++ Rear} + catch + throw:{not_found_source, Type} -> not_found end. -ifdef(TEST). From 505d677685dae9b0cf0fca37d6848170f6b26e7c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 11:43:11 +0800 Subject: [PATCH 10/12] chore: move reload_local_etc_config from emqx_conf to emqx_conf_cli --- apps/emqx/src/emqx_config_handler.erl | 8 +- apps/emqx/src/emqx_schema.erl | 46 +++--- apps/emqx/test/emqx_common_test_helpers.erl | 4 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 2 +- apps/emqx_conf/src/emqx_conf.erl | 101 ------------ apps/emqx_conf/src/emqx_conf_cli.erl | 162 +++++++++++++++---- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 9 +- 7 files changed, 168 insertions(+), 164 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 6948b41e5..80c47d09e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -46,11 +46,10 @@ -export([schema/2]). --define(MOD, module). +-define(MOD, '$mod'). -define(WKEY, '?'). -type handler_name() :: module(). --type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}. -optional_callbacks([ pre_config_update/3, @@ -69,10 +68,7 @@ ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. --type state() :: #{ - handlers := handlers(), - atom() => term() -}. +-type state() :: #{handlers := any()}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 08b902585..453c8ea92 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -382,7 +382,7 @@ fields("persistent_table_mria_opts") -> ]; fields("persistent_session_builtin") -> [ - {"type", sc(hoconsc:enum([builtin]), #{default => builtin, desc => ""})}, + {"type", sc(hoconsc:enum([builtin]), #{default => <<"builtin">>, desc => ""})}, {"session", sc(ref("persistent_table_mria_opts"), #{ desc => ?DESC(persistent_session_builtin_session_table) @@ -548,7 +548,7 @@ fields("mqtt") -> sc( hoconsc:union([integer(), disabled]), #{ - default => disabled, + default => <<"disabled">>, desc => ?DESC(mqtt_server_keepalive) } )}, @@ -575,7 +575,7 @@ fields("mqtt") -> sc( hoconsc:union([range(1, inf), infinity]), #{ - default => infinity, + default => <<"infinity">>, desc => ?DESC(mqtt_max_subscriptions) } )}, @@ -639,7 +639,7 @@ fields("mqtt") -> sc( hoconsc:union([disabled, map()]), #{ - default => disabled, + default => <<"disabled">>, desc => ?DESC(mqtt_mqueue_priorities) } )}, @@ -647,7 +647,7 @@ fields("mqtt") -> sc( hoconsc:enum([highest, lowest]), #{ - default => lowest, + default => <<"lowest">>, desc => ?DESC(mqtt_mqueue_default_priority) } )}, @@ -671,7 +671,7 @@ fields("mqtt") -> sc( hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ - default => disabled, + default => <<"disabled">>, desc => ?DESC(mqtt_peer_cert_as_username) } )}, @@ -679,7 +679,7 @@ fields("mqtt") -> sc( hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ - default => disabled, + default => <<"disabled">>, desc => ?DESC(mqtt_peer_cert_as_clientid) } )} @@ -1224,7 +1224,7 @@ fields("ws_opts") -> sc( hoconsc:enum([single, multiple]), #{ - default => multiple, + default => <<"multiple">>, desc => ?DESC(fields_ws_opts_mqtt_piggyback) } )}, @@ -1248,7 +1248,7 @@ fields("ws_opts") -> sc( hoconsc:union([infinity, integer()]), #{ - default => infinity, + default => <<"infinity">>, desc => ?DESC(fields_ws_opts_max_frame_size) } )}, @@ -1506,7 +1506,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([default, filtered, huffman_only, rle]), #{ - default => default, + default => <<"default">>, desc => ?DESC(fields_deflate_opts_strategy) } )}, @@ -1514,7 +1514,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([takeover, no_takeover]), #{ - default => takeover, + default => <<"takeover">>, desc => ?DESC(fields_deflate_opts_server_context_takeover) } )}, @@ -1522,7 +1522,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([takeover, no_takeover]), #{ - default => takeover, + default => <<"takeover">>, desc => ?DESC(fields_deflate_opts_client_context_takeover) } )}, @@ -1557,7 +1557,7 @@ fields("broker") -> sc( hoconsc:enum([local, leader, quorum, all]), #{ - default => quorum, + default => <<"quorum">>, desc => ?DESC(broker_session_locking_strategy) } )}, @@ -1573,7 +1573,7 @@ fields("broker") -> hash_clientid ]), #{ - default => round_robin, + default => <<"round_robin">>, desc => ?DESC(broker_shared_subscription_strategy) } )}, @@ -1626,7 +1626,7 @@ fields("shared_subscription_group") -> hash_clientid ]), #{ - default => random, + default => <<"random">>, desc => ?DESC(shared_subscription_strategy_enum) } )} @@ -1637,7 +1637,7 @@ fields("broker_perf") -> sc( hoconsc:enum([key, tab, global]), #{ - default => key, + default => <<"key">>, desc => ?DESC(broker_perf_route_lock_type) } )}, @@ -1759,7 +1759,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, duration()]), #{ - default => disabled, + default => <<"disabled">>, desc => ?DESC(sysmon_vm_long_gc) } )}, @@ -1959,7 +1959,7 @@ fields("trace") -> [ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ - default => text, + default => <<"text">>, deprecated => {since, "5.0.22"}, importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(fields_trace_payload_encode) @@ -2048,7 +2048,7 @@ base_listener(Bind) -> atom(), #{ desc => ?DESC(base_listener_zone), - default => 'default' + default => <<"default">> } )}, {"limiter", @@ -2283,7 +2283,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( hoconsc:enum([verify_peer, verify_none]), #{ - default => Df("verify", verify_none), + default => Df("verify", <<"verify_none">>), desc => ?DESC(common_ssl_opts_schema_verify) } )}, @@ -2351,7 +2351,7 @@ common_ssl_opts_schema(Defaults, Type) -> emergency, alert, critical, error, warning, notice, info, debug, none, all ]), #{ - default => notice, + default => <<"notice">>, desc => ?DESC(common_ssl_opts_schema_log_level), importance => ?IMPORTANCE_LOW } @@ -2611,7 +2611,7 @@ authz_fields() -> sc( hoconsc:enum([allow, deny]), #{ - default => allow, + default => <<"allow">>, required => true, desc => ?DESC(fields_authorization_no_match) } @@ -2620,7 +2620,7 @@ authz_fields() -> sc( hoconsc:enum([ignore, disconnect]), #{ - default => ignore, + default => <<"ignore">>, required => true, desc => ?DESC(fields_authorization_deny_action) } diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 1cfc10f74..bf4b2c0ad 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -286,9 +286,9 @@ perform_sanity_checks(_App) -> ok. ensure_config_handler(Module, ConfigPath) -> - #{handlers := Handlers} = sys:get_state(emqx_config_handler), + #{handlers := Handlers} = emqx_config_handler:info(), case emqx_utils_maps:deep_get(ConfigPath, Handlers, not_found) of - #{{mod} := Module} -> ok; + #{'$mod' := Module} -> ok; NotFound -> error({config_handler_missing, ConfigPath, Module, NotFound}) end, ok. diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 194198571..7a2698dd8 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --define(MOD, {mod}). +-define(MOD, '$mod'). -define(WKEY, '?'). -define(CLUSTER_CONF, "/tmp/cluster.conf"). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 483241c37..584a10a8d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -31,7 +31,6 @@ -export([dump_schema/2]). -export([schema_module/0]). -export([gen_example_conf/2]). --export([reload_etc_conf_on_local_node/0]). -export([check_config/2]). %% TODO: move to emqx_dashboard when we stop building api schema at build time @@ -216,94 +215,6 @@ schema_module() -> Value -> list_to_existing_atom(Value) end. -%% @doc Reload etc/emqx.conf to runtime config except for the readonly config --spec reload_etc_conf_on_local_node() -> ok | {error, term()}. -reload_etc_conf_on_local_node() -> - case load_etc_config_file() of - {ok, RawConf} -> - case check_readonly_config(RawConf) of - {ok, Reloaded} -> reload_config(Reloaded); - {error, Error} -> {error, Error} - end; - {error, _Error} -> - {error, bad_hocon_file} - end. - -reload_config(AllConf) -> - Func = fun(Key, Conf, Acc) -> - case emqx:update_config([Key], Conf, #{persistent => false}) of - {ok, _} -> - io:format("Reloaded ~ts config ok~n", [Key]), - Acc; - Error -> - ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), - ?SLOG(error, #{ - msg => "failed_to_reload_etc_config", - key => Key, - value => Conf, - error => Error - }), - Acc#{Key => Error} - end - end, - Res = maps:fold(Func, #{}, AllConf), - case Res =:= #{} of - true -> ok; - false -> {error, Res} - end. - -%% @doc Merge etc/emqx.conf on top of cluster.hocon. -%% For example: -%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, -%% but `authorization.sources` in not in the default emqx.conf file. -%% To make sure all root keys in emqx.conf has a fully merged value. -load_etc_config_file() -> - ConfFiles = emqx_config:config_files(), - Opts = #{format => map, include_dirs => emqx_config:include_dirs()}, - case hocon:files(ConfFiles, Opts) of - {ok, RawConf} -> - HasDeprecatedFile = emqx_config:has_deprecated_file(), - %% Merge etc.conf on top of cluster.hocon, - %% Don't use map deep_merge, use hocon files merge instead. - %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) - Keys = maps:keys(RawConf), - MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles), - {ok, maps:with(Keys, MergedRaw)}; - {error, Error} -> - ?SLOG(error, #{ - msg => "failed_to_read_etc_config", - files => ConfFiles, - error => Error - }), - {error, Error} - end. - -check_readonly_config(Raw) -> - SchemaMod = schema_module(), - RawDefault = emqx_config:fill_defaults(Raw), - case check_config(SchemaMod, RawDefault) of - {ok, CheckedConf} -> - case - lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) - of - [] -> - {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)}; - Error -> - ?SLOG(error, #{ - msg => "failed_to_change_read_only_key_in_etc_config", - read_only_keys => ?READONLY_KEYS, - error => Error - }), - {error, Error} - end; - {error, Error} -> - ?SLOG(error, #{ - msg => "failed_to_check_etc_config", - error => Error - }), - {error, Error} - end. - check_config(Mod, Raw) -> try {_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw), @@ -317,18 +228,6 @@ check_config(Mod, Raw) -> %% Internal functions %%-------------------------------------------------------------------- -filter_changed(Key, ChangedConf) -> - Prev = get([Key], #{}), - New = maps:get(Key, ChangedConf, #{}), - case Prev =/= New of - true -> {true, {Key, changed(New, Prev)}}; - false -> false - end. - -changed(New, Prev) -> - Diff = emqx_utils_maps:diff_maps(New, Prev), - maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). - %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. make_desc_resolver(Lang) -> fun diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 4aae4040b..f2aeec7fb 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -31,6 +31,7 @@ %% kept cluster_call for compatibility -define(CLUSTER_CALL, cluster_call). -define(CONF, conf). +-define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited"). load() -> emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), @@ -55,7 +56,7 @@ conf(["load", Path]) -> conf(["cluster_sync" | Args]) -> admins(Args); conf(["reload"]) -> - emqx_conf:reload_etc_conf_on_local_node(); + reload_etc_conf_on_local_node(); conf(_) -> emqx_ctl:usage(usage_conf() ++ usage_sync()). @@ -190,16 +191,25 @@ load_config(Path, AuthChain) -> {ok, RawConf} -> case check_config(RawConf) of ok -> - maps:foreach(fun(K, V) -> update_config(K, V, AuthChain) end, RawConf); - {error, Reason} when is_list(Reason) -> + lists:foreach( + fun({K, V}) -> update_config(K, V, AuthChain) end, + to_sorted_list(RawConf) + ); + {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), emqx_ctl:warning( "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" ), {error, Reason}; - {error, Reason} when is_map(Reason) -> - emqx_ctl:warning("load ~ts schema check failed~n~p~n", [Path, Reason]), - {error, Reason} + {error, Errors} -> + emqx_ctl:warning("load ~ts schema check failed~n", [Path]), + lists:foreach( + fun({Key, Error}) -> + emqx_ctl:warning("~ts: ~p~n", [Key, Error]) + end, + Errors + ), + {error, Errors} end; {error, Reason} -> emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), @@ -207,14 +217,11 @@ load_config(Path, AuthChain) -> end. update_config(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> - Res = emqx_authz:merge(Conf), - check_res(Key, Res); + check_res(Key, emqx_authz:merge(Conf)); update_config(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME = Key, Conf, "merge") -> - Res = emqx_authn:merge_config(Conf), - check_res(Key, Res); + check_res(Key, emqx_authn:merge_config(Conf)); update_config(Key, Value, _) -> - Res = emqx_conf:update([Key], Value, ?OPTIONS), - check_res(Key, Res). + check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)). check_res(Key, {ok, _}) -> emqx_ctl:print("load ~ts in cluster ok~n", [Key]); check_res(Key, {error, Reason}) -> emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]). @@ -230,24 +237,123 @@ check_keys_is_not_readonly(Conf) -> ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case ReadOnlyKeys -- Keys of ReadOnlyKeys -> ok; - _ -> {error, "update_readonly_keys_prohibited"} + _ -> {error, ?UPDATE_READONLY_KEYS_PROHIBITED} end. check_config_schema(Conf) -> SchemaMod = emqx_conf:schema_module(), - Res = - maps:fold( - fun(Key, Value, Acc) -> - Schema = emqx_config_handler:schema(SchemaMod, [Key]), - case emqx_conf:check_config(Schema, #{Key => Value}) of - {ok, _} -> Acc; - {error, Reason} -> #{Key => Reason} - end - end, - #{}, - Conf - ), - case Res =:= #{} of - true -> ok; - false -> {error, Res} + Fold = fun({Key, Value}, Acc) -> + Schema = emqx_config_handler:schema(SchemaMod, [Key]), + case emqx_conf:check_config(Schema, #{Key => Value}) of + {ok, _} -> Acc; + {error, Reason} -> [{Key, Reason} | Acc] + end + end, + sorted_fold(Fold, Conf). + +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = emqx_config:config_files(), + Opts = #{format => map, include_dirs => emqx_config:include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = emqx_config:has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = emqx_conf:schema_module(), + RawDefault = emqx_config:fill_defaults(Raw), + case emqx_conf:check_config(SchemaMod, RawDefault) of + {ok, CheckedConf} -> + case filter_changed_readonly_keys(CheckedConf) of + [] -> + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], + {ok, maps:without(ReadOnlyKeys, Raw)}; + Error -> + ?SLOG(error, #{ + msg => ?UPDATE_READONLY_KEYS_PROHIBITED, + read_only_keys => ?READONLY_KEYS, + error => Error + }), + {error, Error} + end; + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_etc_config_schema_found", + error => Error + }), + {error, Error} + end. + +reload_config(AllConf) -> + Fold = fun({Key, Conf}, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + emqx_ctl:print("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + emqx_ctl:warning("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + [{Key, Error} | Acc] + end + end, + sorted_fold(Fold, AllConf). + +filter_changed_readonly_keys(Conf) -> + lists:filtermap(fun(Key) -> filter_changed(Key, Conf) end, ?READONLY_KEYS). + +filter_changed(Key, ChangedConf) -> + Prev = emqx_conf:get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + +sorted_fold(Func, Conf) -> + case lists:foldl(Func, [], to_sorted_list(Conf)) of + [] -> ok; + Error -> {error, Error} + end. + +to_sorted_list(Conf) -> + lists:keysort(1, maps:to_list(Conf)). diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl index a485f4edd..0e0cc0127 100644 --- a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -40,18 +40,21 @@ t_load_config(Config) -> ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}), ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), ok = emqx_conf_cli:conf(["load", ConfFile0]), - ?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])), + ?assertEqual(Conf#{<<"sources">> => []}, emqx_conf:get_raw([Authz])), %% remove sources, it will reset to default file source. ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}), ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), ok = emqx_conf_cli:conf(["load", ConfFile1]), Default = [emqx_authz_schema:default_authz()], - ?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])), + ?assertEqual(Conf#{<<"sources">> => Default}, emqx_conf:get_raw([Authz])), %% reset ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}), ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config), ok = emqx_conf_cli:conf(["load", ConfFile2]), - ?assertEqual(Conf, emqx_conf:get_raw([Authz])), + ?assertEqual( + Conf#{<<"sources">> => [emqx_authz_schema:default_authz()]}, + emqx_conf:get_raw([Authz]) + ), ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])), ok. From 346b363cb5b5195768d3e982984c5140215df205 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 15:12:04 +0800 Subject: [PATCH 11/12] test: config SUITE failed --- apps/emqx/test/emqx_config_SUITE.erl | 6 +++--- apps/emqx/test/emqx_config_handler_SUITE.erl | 2 +- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index c87c2c533..5e9284a41 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -63,12 +63,12 @@ t_fill_default_values(C) when is_list(C) -> <<"enable_session_registry">> := true, <<"perf">> := #{ - <<"route_lock_type">> := key, + <<"route_lock_type">> := <<"key">>, <<"trie_compaction">> := true }, <<"route_batch_clean">> := false, - <<"session_locking_strategy">> := quorum, - <<"shared_subscription_strategy">> := round_robin + <<"session_locking_strategy">> := <<"quorum">>, + <<"shared_subscription_strategy">> := <<"round_robin">> } }, WithDefaults diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 7a2698dd8..d2f2faedb 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -99,7 +99,7 @@ t_conflict_handler(_Config) -> %% override ok = emqx_config_handler:add_handler([sysmon], emqx_config_logger), ?assertMatch( - #{handlers := #{sysmon := #{{mod} := emqx_config_logger}}}, + #{handlers := #{sysmon := #{?MOD := emqx_config_logger}}}, emqx_config_handler:info() ), ok. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index f20632414..2f071a828 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -451,7 +451,7 @@ request_for_log(Credential, #{url := Url, method := Method} = State) -> base_url => Url, path_query => PathQuery, headers => Headers, - mody => Body + body => Body } end. From 072f6fac8d36d5cf156af38b0eca80f059e6762e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 10 Jun 2023 19:27:58 +0800 Subject: [PATCH 12/12] chore: update hocon to 0.39.8 --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_schema.erl | 46 +++++++++---------- .../emqx_conf/test/emqx_conf_logger_SUITE.erl | 4 +- mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9dedf3644..070922d22 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 453c8ea92..08b902585 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -382,7 +382,7 @@ fields("persistent_table_mria_opts") -> ]; fields("persistent_session_builtin") -> [ - {"type", sc(hoconsc:enum([builtin]), #{default => <<"builtin">>, desc => ""})}, + {"type", sc(hoconsc:enum([builtin]), #{default => builtin, desc => ""})}, {"session", sc(ref("persistent_table_mria_opts"), #{ desc => ?DESC(persistent_session_builtin_session_table) @@ -548,7 +548,7 @@ fields("mqtt") -> sc( hoconsc:union([integer(), disabled]), #{ - default => <<"disabled">>, + default => disabled, desc => ?DESC(mqtt_server_keepalive) } )}, @@ -575,7 +575,7 @@ fields("mqtt") -> sc( hoconsc:union([range(1, inf), infinity]), #{ - default => <<"infinity">>, + default => infinity, desc => ?DESC(mqtt_max_subscriptions) } )}, @@ -639,7 +639,7 @@ fields("mqtt") -> sc( hoconsc:union([disabled, map()]), #{ - default => <<"disabled">>, + default => disabled, desc => ?DESC(mqtt_mqueue_priorities) } )}, @@ -647,7 +647,7 @@ fields("mqtt") -> sc( hoconsc:enum([highest, lowest]), #{ - default => <<"lowest">>, + default => lowest, desc => ?DESC(mqtt_mqueue_default_priority) } )}, @@ -671,7 +671,7 @@ fields("mqtt") -> sc( hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ - default => <<"disabled">>, + default => disabled, desc => ?DESC(mqtt_peer_cert_as_username) } )}, @@ -679,7 +679,7 @@ fields("mqtt") -> sc( hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ - default => <<"disabled">>, + default => disabled, desc => ?DESC(mqtt_peer_cert_as_clientid) } )} @@ -1224,7 +1224,7 @@ fields("ws_opts") -> sc( hoconsc:enum([single, multiple]), #{ - default => <<"multiple">>, + default => multiple, desc => ?DESC(fields_ws_opts_mqtt_piggyback) } )}, @@ -1248,7 +1248,7 @@ fields("ws_opts") -> sc( hoconsc:union([infinity, integer()]), #{ - default => <<"infinity">>, + default => infinity, desc => ?DESC(fields_ws_opts_max_frame_size) } )}, @@ -1506,7 +1506,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([default, filtered, huffman_only, rle]), #{ - default => <<"default">>, + default => default, desc => ?DESC(fields_deflate_opts_strategy) } )}, @@ -1514,7 +1514,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([takeover, no_takeover]), #{ - default => <<"takeover">>, + default => takeover, desc => ?DESC(fields_deflate_opts_server_context_takeover) } )}, @@ -1522,7 +1522,7 @@ fields("deflate_opts") -> sc( hoconsc:enum([takeover, no_takeover]), #{ - default => <<"takeover">>, + default => takeover, desc => ?DESC(fields_deflate_opts_client_context_takeover) } )}, @@ -1557,7 +1557,7 @@ fields("broker") -> sc( hoconsc:enum([local, leader, quorum, all]), #{ - default => <<"quorum">>, + default => quorum, desc => ?DESC(broker_session_locking_strategy) } )}, @@ -1573,7 +1573,7 @@ fields("broker") -> hash_clientid ]), #{ - default => <<"round_robin">>, + default => round_robin, desc => ?DESC(broker_shared_subscription_strategy) } )}, @@ -1626,7 +1626,7 @@ fields("shared_subscription_group") -> hash_clientid ]), #{ - default => <<"random">>, + default => random, desc => ?DESC(shared_subscription_strategy_enum) } )} @@ -1637,7 +1637,7 @@ fields("broker_perf") -> sc( hoconsc:enum([key, tab, global]), #{ - default => <<"key">>, + default => key, desc => ?DESC(broker_perf_route_lock_type) } )}, @@ -1759,7 +1759,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, duration()]), #{ - default => <<"disabled">>, + default => disabled, desc => ?DESC(sysmon_vm_long_gc) } )}, @@ -1959,7 +1959,7 @@ fields("trace") -> [ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ - default => <<"text">>, + default => text, deprecated => {since, "5.0.22"}, importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(fields_trace_payload_encode) @@ -2048,7 +2048,7 @@ base_listener(Bind) -> atom(), #{ desc => ?DESC(base_listener_zone), - default => <<"default">> + default => 'default' } )}, {"limiter", @@ -2283,7 +2283,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( hoconsc:enum([verify_peer, verify_none]), #{ - default => Df("verify", <<"verify_none">>), + default => Df("verify", verify_none), desc => ?DESC(common_ssl_opts_schema_verify) } )}, @@ -2351,7 +2351,7 @@ common_ssl_opts_schema(Defaults, Type) -> emergency, alert, critical, error, warning, notice, info, debug, none, all ]), #{ - default => <<"notice">>, + default => notice, desc => ?DESC(common_ssl_opts_schema_log_level), importance => ?IMPORTANCE_LOW } @@ -2611,7 +2611,7 @@ authz_fields() -> sc( hoconsc:enum([allow, deny]), #{ - default => <<"allow">>, + default => allow, required => true, desc => ?DESC(fields_authorization_no_match) } @@ -2620,7 +2620,7 @@ authz_fields() -> sc( hoconsc:enum([ignore, disconnect]), #{ - default => <<"ignore">>, + default => ignore, required => true, desc => ?DESC(fields_authorization_deny_action) } diff --git a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl index cc874756d..ba74ed986 100644 --- a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl @@ -62,7 +62,7 @@ end_per_suite(_Config) -> t_log_conf(_Conf) -> FileExpect = #{ <<"enable">> => true, - <<"formatter">> => text, + <<"formatter">> => <<"text">>, <<"level">> => <<"info">>, <<"rotation_count">> => 10, <<"rotation_size">> => <<"50MB">>, @@ -73,7 +73,7 @@ t_log_conf(_Conf) -> <<"console">> => #{ <<"enable">> => true, - <<"formatter">> => text, + <<"formatter">> => <<"text">>, <<"level">> => <<"debug">>, <<"time_offset">> => <<"system">> }, diff --git a/mix.exs b/mix.exs index 67fdee443..2058ea3fa 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.7", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.8", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 8bbba5b96..7cfc5aa81 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.7"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.8"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}