diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 75a4a7714..64cd4687b 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -17,9 +17,6 @@ -ifndef(EMQX_HRL). -define(EMQX_HRL, true). -%% Config --define(READ_ONLY_KEYS, [cluster, rpc, node]). - %% Shard %%-------------------------------------------------------------------- -define(COMMON_SHARD, emqx_common_shard). diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 790f6040d..2f7a4350a 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -35,7 +35,8 @@ save_to_app_env/1, save_to_config_map/2, save_to_override_conf/3, - reload_etc_conf_on_local_node/0 + config_files/0, + include_dirs/0 ]). -export([merge_envs/2]). @@ -91,6 +92,7 @@ ]). -export([ensure_atom_conf_path/2]). +-export([load_config_files/2]). -ifdef(TEST). -export([erase_all/0, backup_and_write/2]). @@ -977,94 +979,5 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) -> put_config_post_change_actions(_Key, _NewValue) -> ok. -%% @doc Reload etc/emqx.conf to runtime config except for the readonly config --spec reload_etc_conf_on_local_node() -> ok | {error, term()}. -reload_etc_conf_on_local_node() -> - case load_etc_config_file() of - {ok, RawConf} -> - case check_readonly_config(RawConf) of - {ok, Reloaded} -> reload_config(Reloaded); - {error, Error} -> {error, Error} - end; - {error, _Error} -> - {error, bad_hocon_file} - end. - -reload_config(AllConf) -> - Func = fun(Key, Conf, Acc) -> - case emqx:update_config([Key], Conf, #{persistent => false}) of - {ok, _} -> - io:format("Reloaded ~ts config ok~n", [Key]), - Acc; - Error -> - ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), - ?SLOG(error, #{ - msg => "failed_to_reload_etc_config", - key => Key, - value => Conf, - error => Error - }), - Acc#{Key => Error} - end - end, - Res = maps:fold(Func, #{}, AllConf), - case Res =:= #{} of - true -> ok; - false -> {error, Res} - end. - -%% @doc Merge etc/emqx.conf on top of cluster.hocon. -%% For example: -%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, -%% but `authorization.sources` in not in the default emqx.conf file. -%% To make sure all root keys in emqx.conf has a fully merged value. -load_etc_config_file() -> - ConfFiles = config_files(), - Opts = #{format => map, include_dirs => include_dirs()}, - case hocon:files(ConfFiles, Opts) of - {ok, RawConf} -> - HasDeprecatedFile = has_deprecated_file(), - %% Merge etc.conf on top of cluster.hocon, - %% Don't use map deep_merge, use hocon files merge instead. - %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) - Keys = maps:keys(RawConf), - MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles), - {ok, maps:with(Keys, MergedRaw)}; - {error, Error} -> - ?SLOG(error, #{ - msg => "failed_to_read_etc_config", - files => ConfFiles, - error => Error - }), - {error, Error} - end. - -check_readonly_config(Raw) -> - SchemaMod = emqx_conf:schema_module(), - {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}), - case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of - [] -> - {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)}; - Error -> - ?SLOG(error, #{ - msg => "failed_to_change_read_only_key_in_etc_config", - read_only_keys => ?READ_ONLY_KEYS, - error => Error - }), - {error, Error} - end. - -filter_changed(Key, ChangedConf) -> - Prev = get([Key], #{}), - New = maps:get(Key, ChangedConf, #{}), - case Prev =/= New of - true -> {true, {Key, changed(New, Prev)}}; - false -> false - end. - -changed(New, Prev) -> - Diff = emqx_utils_maps:diff_maps(New, Prev), - maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). - config_files() -> application:get_env(emqx, config_files, []). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 9dd46c0c2..b4c943072 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -166,8 +166,9 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) -> do_pre_config_replace(Conf, Conf) -> Conf; do_pre_config_replace(NewConf, OldConf) -> - NewSources = maps:get(<<"sources">>, NewConf, []), - OldSources = maps:get(<<"sources">>, OldConf, []), + Default = [emqx_authz_schema:default_authz()], + NewSources = maps:get(<<"sources">>, NewConf, Default), + OldSources = maps:get(<<"sources">>, OldConf, Default), NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), NewConf#{<<"sources">> => NewSources1}. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 8e847b93e..b19c62441 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,7 +42,8 @@ -export([ headers_no_content_type/1, - headers/1 + headers/1, + default_authz/0 ]). %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index 0297fddf7..26042d62c 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -34,4 +34,6 @@ tnx_id :: pos_integer() | '$1' }). +-define(READONLY_KEYS, [cluster, rpc, node]). + -endif. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d2e45581d..b9e5e17fa 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -19,6 +19,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/emqx_schema.hrl"). +-include("emqx_conf.hrl"). -export([add_handler/2, remove_handler/1]). -export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]). @@ -30,6 +31,7 @@ -export([dump_schema/2]). -export([schema_module/0]). -export([gen_example_conf/2]). +-export([reload_etc_conf_on_local_node/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ @@ -213,10 +215,100 @@ schema_module() -> Value -> list_to_existing_atom(Value) end. +%% @doc Reload etc/emqx.conf to runtime config except for the readonly config +-spec reload_etc_conf_on_local_node() -> ok | {error, term()}. +reload_etc_conf_on_local_node() -> + case load_etc_config_file() of + {ok, RawConf} -> + case check_readonly_config(RawConf) of + {ok, Reloaded} -> reload_config(Reloaded); + {error, Error} -> {error, Error} + end; + {error, _Error} -> + {error, bad_hocon_file} + end. + +reload_config(AllConf) -> + Func = fun(Key, Conf, Acc) -> + case emqx:update_config([Key], Conf, #{persistent => false}) of + {ok, _} -> + io:format("Reloaded ~ts config ok~n", [Key]), + Acc; + Error -> + ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]), + ?SLOG(error, #{ + msg => "failed_to_reload_etc_config", + key => Key, + value => Conf, + error => Error + }), + Acc#{Key => Error} + end + end, + Res = maps:fold(Func, #{}, AllConf), + case Res =:= #{} of + true -> ok; + false -> {error, Res} + end. + +%% @doc Merge etc/emqx.conf on top of cluster.hocon. +%% For example: +%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard, +%% but `authorization.sources` in not in the default emqx.conf file. +%% To make sure all root keys in emqx.conf has a fully merged value. +load_etc_config_file() -> + ConfFiles = emqx_config:config_files(), + Opts = #{format => map, include_dirs => emqx_config:include_dirs()}, + case hocon:files(ConfFiles, Opts) of + {ok, RawConf} -> + HasDeprecatedFile = emqx_config:has_deprecated_file(), + %% Merge etc.conf on top of cluster.hocon, + %% Don't use map deep_merge, use hocon files merge instead. + %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null) + Keys = maps:keys(RawConf), + MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles), + {ok, maps:with(Keys, MergedRaw)}; + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_read_etc_config", + files => ConfFiles, + error => Error + }), + {error, Error} + end. + +check_readonly_config(Raw) -> + SchemaMod = schema_module(), + RawDefault = emqx_config:fill_defaults(Raw), + {_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault), + case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of + [] -> + {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)}; + Error -> + ?SLOG(error, #{ + msg => "failed_to_change_read_only_key_in_etc_config", + read_only_keys => ?READONLY_KEYS, + error => Error + }), + {error, Error} + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +filter_changed(Key, ChangedConf) -> + Prev = get([Key], #{}), + New = maps:get(Key, ChangedConf, #{}), + case Prev =/= New of + true -> {true, {Key, changed(New, Prev)}}; + false -> false + end. + +changed(New, Prev) -> + Diff = emqx_utils_maps:diff_maps(New, Prev), + maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)). + %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. make_desc_resolver(Lang) -> fun diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 8d74c4cd9..eb8057fc9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_conf_cli). --include_lib("emqx/include/emqx.hrl"). +-include("emqx_conf.hrl"). -export([ load/0, admins/1, @@ -47,6 +47,8 @@ conf(["load", Path]) -> load_config(Path); conf(["cluster_sync" | Args]) -> admins(Args); +conf(["reload"]) -> + emqx_conf:reload_etc_conf_on_local_node(); conf(_) -> emqx_ctl:usage(usage_conf() ++ usage_sync()). @@ -175,6 +177,9 @@ get_config(Key) -> -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). load_config(Path) -> case hocon:files([Path]) of + {ok, RawConf} when RawConf =:= #{} -> + emqx_ctl:warning("load ~ts is empty~n", [Path]), + {error, empty_hocon_file}; {ok, RawConf} -> case check_config_keys(RawConf) of ok -> @@ -200,8 +205,8 @@ update_config(Key, Value) -> end. check_config_keys(Conf) -> Keys = maps:keys(Conf), - ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS], + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case ReadOnlyKeys -- Keys of ReadOnlyKeys -> ok; - _ -> {error, "update_read_only_keys_prohibited"} + _ -> {error, "update_readonly_keys_prohibited"} end. diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl new file mode 100644 index 000000000..3ca90646b --- /dev/null +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_conf_cli_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include("emqx_conf.hrl"). +-import(emqx_config_SUITE, [prepare_conf_file/3]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]). + +t_load_config(Config) -> + Authz = authorization, + Conf = emqx_conf:get_raw([Authz]), + %% set sources to [] + ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ok = emqx_conf_cli:conf(["load", ConfFile0]), + ?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])), + %% remove sources, it will reset to default file source. + ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}), + ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config), + ok = emqx_conf_cli:conf(["load", ConfFile1]), + Default = [emqx_authz_schema:default_authz()], + ?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])), + %% reset + ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}), + ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config), + ok = emqx_conf_cli:conf(["load", ConfFile2]), + ?assertEqual(Conf, emqx_conf:get_raw([Authz])), + ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])), + ok. + +t_load_readonly(Config) -> + Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])}, + lists:foreach( + fun(Key) -> + Conf = emqx_conf:get_raw([Key]), + ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}), + ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), + ?assertEqual( + {error, "update_readonly_keys_prohibited"}, + emqx_conf_cli:conf(["load", ConfFile0]) + ) + end, + ?READONLY_KEYS + ), + ok.