fix: load authz's default sources

This commit is contained in:
zhongwencool 2023-06-08 23:55:11 +08:00
parent c819ac27f4
commit 994db58cbf
8 changed files with 182 additions and 99 deletions

View File

@ -17,9 +17,6 @@
-ifndef(EMQX_HRL). -ifndef(EMQX_HRL).
-define(EMQX_HRL, true). -define(EMQX_HRL, true).
%% Config
-define(READ_ONLY_KEYS, [cluster, rpc, node]).
%% Shard %% Shard
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-define(COMMON_SHARD, emqx_common_shard). -define(COMMON_SHARD, emqx_common_shard).

View File

@ -35,7 +35,8 @@
save_to_app_env/1, save_to_app_env/1,
save_to_config_map/2, save_to_config_map/2,
save_to_override_conf/3, save_to_override_conf/3,
reload_etc_conf_on_local_node/0 config_files/0,
include_dirs/0
]). ]).
-export([merge_envs/2]). -export([merge_envs/2]).
@ -91,6 +92,7 @@
]). ]).
-export([ensure_atom_conf_path/2]). -export([ensure_atom_conf_path/2]).
-export([load_config_files/2]).
-ifdef(TEST). -ifdef(TEST).
-export([erase_all/0, backup_and_write/2]). -export([erase_all/0, backup_and_write/2]).
@ -977,94 +979,5 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
put_config_post_change_actions(_Key, _NewValue) -> put_config_post_change_actions(_Key, _NewValue) ->
ok. ok.
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
reload_etc_conf_on_local_node() ->
case load_etc_config_file() of
{ok, RawConf} ->
case check_readonly_config(RawConf) of
{ok, Reloaded} -> reload_config(Reloaded);
{error, Error} -> {error, Error}
end;
{error, _Error} ->
{error, bad_hocon_file}
end.
reload_config(AllConf) ->
Func = fun(Key, Conf, Acc) ->
case emqx:update_config([Key], Conf, #{persistent => false}) of
{ok, _} ->
io:format("Reloaded ~ts config ok~n", [Key]),
Acc;
Error ->
?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
?SLOG(error, #{
msg => "failed_to_reload_etc_config",
key => Key,
value => Conf,
error => Error
}),
Acc#{Key => Error}
end
end,
Res = maps:fold(Func, #{}, AllConf),
case Res =:= #{} of
true -> ok;
false -> {error, Res}
end.
%% @doc Merge etc/emqx.conf on top of cluster.hocon.
%% For example:
%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
%% but `authorization.sources` in not in the default emqx.conf file.
%% To make sure all root keys in emqx.conf has a fully merged value.
load_etc_config_file() ->
ConfFiles = config_files(),
Opts = #{format => map, include_dirs => include_dirs()},
case hocon:files(ConfFiles, Opts) of
{ok, RawConf} ->
HasDeprecatedFile = has_deprecated_file(),
%% Merge etc.conf on top of cluster.hocon,
%% Don't use map deep_merge, use hocon files merge instead.
%% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
Keys = maps:keys(RawConf),
MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles),
{ok, maps:with(Keys, MergedRaw)};
{error, Error} ->
?SLOG(error, #{
msg => "failed_to_read_etc_config",
files => ConfFiles,
error => Error
}),
{error, Error}
end.
check_readonly_config(Raw) ->
SchemaMod = emqx_conf:schema_module(),
{_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}),
case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of
[] ->
{ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)};
Error ->
?SLOG(error, #{
msg => "failed_to_change_read_only_key_in_etc_config",
read_only_keys => ?READ_ONLY_KEYS,
error => Error
}),
{error, Error}
end.
filter_changed(Key, ChangedConf) ->
Prev = get([Key], #{}),
New = maps:get(Key, ChangedConf, #{}),
case Prev =/= New of
true -> {true, {Key, changed(New, Prev)}};
false -> false
end.
changed(New, Prev) ->
Diff = emqx_utils_maps:diff_maps(New, Prev),
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
config_files() -> config_files() ->
application:get_env(emqx, config_files, []). application:get_env(emqx, config_files, []).

View File

@ -166,8 +166,9 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
do_pre_config_replace(Conf, Conf) -> do_pre_config_replace(Conf, Conf) ->
Conf; Conf;
do_pre_config_replace(NewConf, OldConf) -> do_pre_config_replace(NewConf, OldConf) ->
NewSources = maps:get(<<"sources">>, NewConf, []), Default = [emqx_authz_schema:default_authz()],
OldSources = maps:get(<<"sources">>, OldConf, []), NewSources = maps:get(<<"sources">>, NewConf, Default),
OldSources = maps:get(<<"sources">>, OldConf, Default),
NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources), NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
NewConf#{<<"sources">> => NewSources1}. NewConf#{<<"sources">> => NewSources1}.

View File

@ -42,7 +42,8 @@
-export([ -export([
headers_no_content_type/1, headers_no_content_type/1,
headers/1 headers/1,
default_authz/0
]). ]).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------

View File

@ -34,4 +34,6 @@
tnx_id :: pos_integer() | '$1' tnx_id :: pos_integer() | '$1'
}). }).
-define(READONLY_KEYS, [cluster, rpc, node]).
-endif. -endif.

View File

@ -19,6 +19,7 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-include_lib("hocon/include/hoconsc.hrl"). -include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx_schema.hrl"). -include_lib("emqx/include/emqx_schema.hrl").
-include("emqx_conf.hrl").
-export([add_handler/2, remove_handler/1]). -export([add_handler/2, remove_handler/1]).
-export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]). -export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]).
@ -30,6 +31,7 @@
-export([dump_schema/2]). -export([dump_schema/2]).
-export([schema_module/0]). -export([schema_module/0]).
-export([gen_example_conf/2]). -export([gen_example_conf/2]).
-export([reload_etc_conf_on_local_node/0]).
%% TODO: move to emqx_dashboard when we stop building api schema at build time %% TODO: move to emqx_dashboard when we stop building api schema at build time
-export([ -export([
@ -213,10 +215,100 @@ schema_module() ->
Value -> list_to_existing_atom(Value) Value -> list_to_existing_atom(Value)
end. end.
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
reload_etc_conf_on_local_node() ->
case load_etc_config_file() of
{ok, RawConf} ->
case check_readonly_config(RawConf) of
{ok, Reloaded} -> reload_config(Reloaded);
{error, Error} -> {error, Error}
end;
{error, _Error} ->
{error, bad_hocon_file}
end.
reload_config(AllConf) ->
Func = fun(Key, Conf, Acc) ->
case emqx:update_config([Key], Conf, #{persistent => false}) of
{ok, _} ->
io:format("Reloaded ~ts config ok~n", [Key]),
Acc;
Error ->
?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
?SLOG(error, #{
msg => "failed_to_reload_etc_config",
key => Key,
value => Conf,
error => Error
}),
Acc#{Key => Error}
end
end,
Res = maps:fold(Func, #{}, AllConf),
case Res =:= #{} of
true -> ok;
false -> {error, Res}
end.
%% @doc Merge etc/emqx.conf on top of cluster.hocon.
%% For example:
%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
%% but `authorization.sources` in not in the default emqx.conf file.
%% To make sure all root keys in emqx.conf has a fully merged value.
load_etc_config_file() ->
ConfFiles = emqx_config:config_files(),
Opts = #{format => map, include_dirs => emqx_config:include_dirs()},
case hocon:files(ConfFiles, Opts) of
{ok, RawConf} ->
HasDeprecatedFile = emqx_config:has_deprecated_file(),
%% Merge etc.conf on top of cluster.hocon,
%% Don't use map deep_merge, use hocon files merge instead.
%% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
Keys = maps:keys(RawConf),
MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles),
{ok, maps:with(Keys, MergedRaw)};
{error, Error} ->
?SLOG(error, #{
msg => "failed_to_read_etc_config",
files => ConfFiles,
error => Error
}),
{error, Error}
end.
check_readonly_config(Raw) ->
SchemaMod = schema_module(),
RawDefault = emqx_config:fill_defaults(Raw),
{_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault),
case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of
[] ->
{ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)};
Error ->
?SLOG(error, #{
msg => "failed_to_change_read_only_key_in_etc_config",
read_only_keys => ?READONLY_KEYS,
error => Error
}),
{error, Error}
end.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Internal functions %% Internal functions
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
filter_changed(Key, ChangedConf) ->
Prev = get([Key], #{}),
New = maps:get(Key, ChangedConf, #{}),
case Prev =/= New of
true -> {true, {Key, changed(New, Prev)}};
false -> false
end.
changed(New, Prev) ->
Diff = emqx_utils_maps:diff_maps(New, Prev),
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump.
make_desc_resolver(Lang) -> make_desc_resolver(Lang) ->
fun fun

View File

@ -15,7 +15,7 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-module(emqx_conf_cli). -module(emqx_conf_cli).
-include_lib("emqx/include/emqx.hrl"). -include("emqx_conf.hrl").
-export([ -export([
load/0, load/0,
admins/1, admins/1,
@ -47,6 +47,8 @@ conf(["load", Path]) ->
load_config(Path); load_config(Path);
conf(["cluster_sync" | Args]) -> conf(["cluster_sync" | Args]) ->
admins(Args); admins(Args);
conf(["reload"]) ->
emqx_conf:reload_etc_conf_on_local_node();
conf(_) -> conf(_) ->
emqx_ctl:usage(usage_conf() ++ usage_sync()). emqx_ctl:usage(usage_conf() ++ usage_sync()).
@ -175,6 +177,9 @@ get_config(Key) ->
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
load_config(Path) -> load_config(Path) ->
case hocon:files([Path]) of case hocon:files([Path]) of
{ok, RawConf} when RawConf =:= #{} ->
emqx_ctl:warning("load ~ts is empty~n", [Path]),
{error, empty_hocon_file};
{ok, RawConf} -> {ok, RawConf} ->
case check_config_keys(RawConf) of case check_config_keys(RawConf) of
ok -> ok ->
@ -200,8 +205,8 @@ update_config(Key, Value) ->
end. end.
check_config_keys(Conf) -> check_config_keys(Conf) ->
Keys = maps:keys(Conf), Keys = maps:keys(Conf),
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS], ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
case ReadOnlyKeys -- Keys of case ReadOnlyKeys -- Keys of
ReadOnlyKeys -> ok; ReadOnlyKeys -> ok;
_ -> {error, "update_read_only_keys_prohibited"} _ -> {error, "update_readonly_keys_prohibited"}
end. end.

View File

@ -0,0 +1,72 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_conf_cli_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_conf.hrl").
-import(emqx_config_SUITE, [prepare_conf_file/3]).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]),
Config.
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
t_load_config(Config) ->
Authz = authorization,
Conf = emqx_conf:get_raw([Authz]),
%% set sources to []
ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
ok = emqx_conf_cli:conf(["load", ConfFile0]),
?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])),
%% remove sources, it will reset to default file source.
ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
ok = emqx_conf_cli:conf(["load", ConfFile1]),
Default = [emqx_authz_schema:default_authz()],
?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])),
%% reset
ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
ok = emqx_conf_cli:conf(["load", ConfFile2]),
?assertEqual(Conf, emqx_conf:get_raw([Authz])),
?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])),
ok.
t_load_readonly(Config) ->
Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])},
lists:foreach(
fun(Key) ->
Conf = emqx_conf:get_raw([Key]),
ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}),
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
?assertEqual(
{error, "update_readonly_keys_prohibited"},
emqx_conf_cli:conf(["load", ConfFile0])
)
end,
?READONLY_KEYS
),
ok.