fix: load authz's default sources
This commit is contained in:
parent
c819ac27f4
commit
994db58cbf
|
@ -17,9 +17,6 @@
|
||||||
-ifndef(EMQX_HRL).
|
-ifndef(EMQX_HRL).
|
||||||
-define(EMQX_HRL, true).
|
-define(EMQX_HRL, true).
|
||||||
|
|
||||||
%% Config
|
|
||||||
-define(READ_ONLY_KEYS, [cluster, rpc, node]).
|
|
||||||
|
|
||||||
%% Shard
|
%% Shard
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
-define(COMMON_SHARD, emqx_common_shard).
|
-define(COMMON_SHARD, emqx_common_shard).
|
||||||
|
|
|
@ -35,7 +35,8 @@
|
||||||
save_to_app_env/1,
|
save_to_app_env/1,
|
||||||
save_to_config_map/2,
|
save_to_config_map/2,
|
||||||
save_to_override_conf/3,
|
save_to_override_conf/3,
|
||||||
reload_etc_conf_on_local_node/0
|
config_files/0,
|
||||||
|
include_dirs/0
|
||||||
]).
|
]).
|
||||||
-export([merge_envs/2]).
|
-export([merge_envs/2]).
|
||||||
|
|
||||||
|
@ -91,6 +92,7 @@
|
||||||
]).
|
]).
|
||||||
|
|
||||||
-export([ensure_atom_conf_path/2]).
|
-export([ensure_atom_conf_path/2]).
|
||||||
|
-export([load_config_files/2]).
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
-export([erase_all/0, backup_and_write/2]).
|
-export([erase_all/0, backup_and_write/2]).
|
||||||
|
@ -977,94 +979,5 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
|
||||||
put_config_post_change_actions(_Key, _NewValue) ->
|
put_config_post_change_actions(_Key, _NewValue) ->
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
|
|
||||||
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
|
|
||||||
reload_etc_conf_on_local_node() ->
|
|
||||||
case load_etc_config_file() of
|
|
||||||
{ok, RawConf} ->
|
|
||||||
case check_readonly_config(RawConf) of
|
|
||||||
{ok, Reloaded} -> reload_config(Reloaded);
|
|
||||||
{error, Error} -> {error, Error}
|
|
||||||
end;
|
|
||||||
{error, _Error} ->
|
|
||||||
{error, bad_hocon_file}
|
|
||||||
end.
|
|
||||||
|
|
||||||
reload_config(AllConf) ->
|
|
||||||
Func = fun(Key, Conf, Acc) ->
|
|
||||||
case emqx:update_config([Key], Conf, #{persistent => false}) of
|
|
||||||
{ok, _} ->
|
|
||||||
io:format("Reloaded ~ts config ok~n", [Key]),
|
|
||||||
Acc;
|
|
||||||
Error ->
|
|
||||||
?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
|
|
||||||
?SLOG(error, #{
|
|
||||||
msg => "failed_to_reload_etc_config",
|
|
||||||
key => Key,
|
|
||||||
value => Conf,
|
|
||||||
error => Error
|
|
||||||
}),
|
|
||||||
Acc#{Key => Error}
|
|
||||||
end
|
|
||||||
end,
|
|
||||||
Res = maps:fold(Func, #{}, AllConf),
|
|
||||||
case Res =:= #{} of
|
|
||||||
true -> ok;
|
|
||||||
false -> {error, Res}
|
|
||||||
end.
|
|
||||||
|
|
||||||
%% @doc Merge etc/emqx.conf on top of cluster.hocon.
|
|
||||||
%% For example:
|
|
||||||
%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
|
|
||||||
%% but `authorization.sources` in not in the default emqx.conf file.
|
|
||||||
%% To make sure all root keys in emqx.conf has a fully merged value.
|
|
||||||
load_etc_config_file() ->
|
|
||||||
ConfFiles = config_files(),
|
|
||||||
Opts = #{format => map, include_dirs => include_dirs()},
|
|
||||||
case hocon:files(ConfFiles, Opts) of
|
|
||||||
{ok, RawConf} ->
|
|
||||||
HasDeprecatedFile = has_deprecated_file(),
|
|
||||||
%% Merge etc.conf on top of cluster.hocon,
|
|
||||||
%% Don't use map deep_merge, use hocon files merge instead.
|
|
||||||
%% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
|
|
||||||
Keys = maps:keys(RawConf),
|
|
||||||
MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles),
|
|
||||||
{ok, maps:with(Keys, MergedRaw)};
|
|
||||||
{error, Error} ->
|
|
||||||
?SLOG(error, #{
|
|
||||||
msg => "failed_to_read_etc_config",
|
|
||||||
files => ConfFiles,
|
|
||||||
error => Error
|
|
||||||
}),
|
|
||||||
{error, Error}
|
|
||||||
end.
|
|
||||||
|
|
||||||
check_readonly_config(Raw) ->
|
|
||||||
SchemaMod = emqx_conf:schema_module(),
|
|
||||||
{_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}),
|
|
||||||
case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of
|
|
||||||
[] ->
|
|
||||||
{ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)};
|
|
||||||
Error ->
|
|
||||||
?SLOG(error, #{
|
|
||||||
msg => "failed_to_change_read_only_key_in_etc_config",
|
|
||||||
read_only_keys => ?READ_ONLY_KEYS,
|
|
||||||
error => Error
|
|
||||||
}),
|
|
||||||
{error, Error}
|
|
||||||
end.
|
|
||||||
|
|
||||||
filter_changed(Key, ChangedConf) ->
|
|
||||||
Prev = get([Key], #{}),
|
|
||||||
New = maps:get(Key, ChangedConf, #{}),
|
|
||||||
case Prev =/= New of
|
|
||||||
true -> {true, {Key, changed(New, Prev)}};
|
|
||||||
false -> false
|
|
||||||
end.
|
|
||||||
|
|
||||||
changed(New, Prev) ->
|
|
||||||
Diff = emqx_utils_maps:diff_maps(New, Prev),
|
|
||||||
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
|
|
||||||
|
|
||||||
config_files() ->
|
config_files() ->
|
||||||
application:get_env(emqx, config_files, []).
|
application:get_env(emqx, config_files, []).
|
||||||
|
|
|
@ -166,8 +166,9 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
|
||||||
do_pre_config_replace(Conf, Conf) ->
|
do_pre_config_replace(Conf, Conf) ->
|
||||||
Conf;
|
Conf;
|
||||||
do_pre_config_replace(NewConf, OldConf) ->
|
do_pre_config_replace(NewConf, OldConf) ->
|
||||||
NewSources = maps:get(<<"sources">>, NewConf, []),
|
Default = [emqx_authz_schema:default_authz()],
|
||||||
OldSources = maps:get(<<"sources">>, OldConf, []),
|
NewSources = maps:get(<<"sources">>, NewConf, Default),
|
||||||
|
OldSources = maps:get(<<"sources">>, OldConf, Default),
|
||||||
NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
|
NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
|
||||||
NewConf#{<<"sources">> => NewSources1}.
|
NewConf#{<<"sources">> => NewSources1}.
|
||||||
|
|
||||||
|
|
|
@ -42,7 +42,8 @@
|
||||||
|
|
||||||
-export([
|
-export([
|
||||||
headers_no_content_type/1,
|
headers_no_content_type/1,
|
||||||
headers/1
|
headers/1,
|
||||||
|
default_authz/0
|
||||||
]).
|
]).
|
||||||
|
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
|
|
|
@ -34,4 +34,6 @@
|
||||||
tnx_id :: pos_integer() | '$1'
|
tnx_id :: pos_integer() | '$1'
|
||||||
}).
|
}).
|
||||||
|
|
||||||
|
-define(READONLY_KEYS, [cluster, rpc, node]).
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
-include_lib("emqx/include/logger.hrl").
|
-include_lib("emqx/include/logger.hrl").
|
||||||
-include_lib("hocon/include/hoconsc.hrl").
|
-include_lib("hocon/include/hoconsc.hrl").
|
||||||
-include_lib("emqx/include/emqx_schema.hrl").
|
-include_lib("emqx/include/emqx_schema.hrl").
|
||||||
|
-include("emqx_conf.hrl").
|
||||||
|
|
||||||
-export([add_handler/2, remove_handler/1]).
|
-export([add_handler/2, remove_handler/1]).
|
||||||
-export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]).
|
-export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]).
|
||||||
|
@ -30,6 +31,7 @@
|
||||||
-export([dump_schema/2]).
|
-export([dump_schema/2]).
|
||||||
-export([schema_module/0]).
|
-export([schema_module/0]).
|
||||||
-export([gen_example_conf/2]).
|
-export([gen_example_conf/2]).
|
||||||
|
-export([reload_etc_conf_on_local_node/0]).
|
||||||
|
|
||||||
%% TODO: move to emqx_dashboard when we stop building api schema at build time
|
%% TODO: move to emqx_dashboard when we stop building api schema at build time
|
||||||
-export([
|
-export([
|
||||||
|
@ -213,10 +215,100 @@ schema_module() ->
|
||||||
Value -> list_to_existing_atom(Value)
|
Value -> list_to_existing_atom(Value)
|
||||||
end.
|
end.
|
||||||
|
|
||||||
|
%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
|
||||||
|
-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
|
||||||
|
reload_etc_conf_on_local_node() ->
|
||||||
|
case load_etc_config_file() of
|
||||||
|
{ok, RawConf} ->
|
||||||
|
case check_readonly_config(RawConf) of
|
||||||
|
{ok, Reloaded} -> reload_config(Reloaded);
|
||||||
|
{error, Error} -> {error, Error}
|
||||||
|
end;
|
||||||
|
{error, _Error} ->
|
||||||
|
{error, bad_hocon_file}
|
||||||
|
end.
|
||||||
|
|
||||||
|
reload_config(AllConf) ->
|
||||||
|
Func = fun(Key, Conf, Acc) ->
|
||||||
|
case emqx:update_config([Key], Conf, #{persistent => false}) of
|
||||||
|
{ok, _} ->
|
||||||
|
io:format("Reloaded ~ts config ok~n", [Key]),
|
||||||
|
Acc;
|
||||||
|
Error ->
|
||||||
|
?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
|
||||||
|
?SLOG(error, #{
|
||||||
|
msg => "failed_to_reload_etc_config",
|
||||||
|
key => Key,
|
||||||
|
value => Conf,
|
||||||
|
error => Error
|
||||||
|
}),
|
||||||
|
Acc#{Key => Error}
|
||||||
|
end
|
||||||
|
end,
|
||||||
|
Res = maps:fold(Func, #{}, AllConf),
|
||||||
|
case Res =:= #{} of
|
||||||
|
true -> ok;
|
||||||
|
false -> {error, Res}
|
||||||
|
end.
|
||||||
|
|
||||||
|
%% @doc Merge etc/emqx.conf on top of cluster.hocon.
|
||||||
|
%% For example:
|
||||||
|
%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
|
||||||
|
%% but `authorization.sources` in not in the default emqx.conf file.
|
||||||
|
%% To make sure all root keys in emqx.conf has a fully merged value.
|
||||||
|
load_etc_config_file() ->
|
||||||
|
ConfFiles = emqx_config:config_files(),
|
||||||
|
Opts = #{format => map, include_dirs => emqx_config:include_dirs()},
|
||||||
|
case hocon:files(ConfFiles, Opts) of
|
||||||
|
{ok, RawConf} ->
|
||||||
|
HasDeprecatedFile = emqx_config:has_deprecated_file(),
|
||||||
|
%% Merge etc.conf on top of cluster.hocon,
|
||||||
|
%% Don't use map deep_merge, use hocon files merge instead.
|
||||||
|
%% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
|
||||||
|
Keys = maps:keys(RawConf),
|
||||||
|
MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles),
|
||||||
|
{ok, maps:with(Keys, MergedRaw)};
|
||||||
|
{error, Error} ->
|
||||||
|
?SLOG(error, #{
|
||||||
|
msg => "failed_to_read_etc_config",
|
||||||
|
files => ConfFiles,
|
||||||
|
error => Error
|
||||||
|
}),
|
||||||
|
{error, Error}
|
||||||
|
end.
|
||||||
|
|
||||||
|
check_readonly_config(Raw) ->
|
||||||
|
SchemaMod = schema_module(),
|
||||||
|
RawDefault = emqx_config:fill_defaults(Raw),
|
||||||
|
{_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault),
|
||||||
|
case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of
|
||||||
|
[] ->
|
||||||
|
{ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)};
|
||||||
|
Error ->
|
||||||
|
?SLOG(error, #{
|
||||||
|
msg => "failed_to_change_read_only_key_in_etc_config",
|
||||||
|
read_only_keys => ?READONLY_KEYS,
|
||||||
|
error => Error
|
||||||
|
}),
|
||||||
|
{error, Error}
|
||||||
|
end.
|
||||||
|
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
%% Internal functions
|
%% Internal functions
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
|
|
||||||
|
filter_changed(Key, ChangedConf) ->
|
||||||
|
Prev = get([Key], #{}),
|
||||||
|
New = maps:get(Key, ChangedConf, #{}),
|
||||||
|
case Prev =/= New of
|
||||||
|
true -> {true, {Key, changed(New, Prev)}};
|
||||||
|
false -> false
|
||||||
|
end.
|
||||||
|
|
||||||
|
changed(New, Prev) ->
|
||||||
|
Diff = emqx_utils_maps:diff_maps(New, Prev),
|
||||||
|
maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
|
||||||
|
|
||||||
%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump.
|
%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump.
|
||||||
make_desc_resolver(Lang) ->
|
make_desc_resolver(Lang) ->
|
||||||
fun
|
fun
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
|
|
||||||
-module(emqx_conf_cli).
|
-module(emqx_conf_cli).
|
||||||
-include_lib("emqx/include/emqx.hrl").
|
-include("emqx_conf.hrl").
|
||||||
-export([
|
-export([
|
||||||
load/0,
|
load/0,
|
||||||
admins/1,
|
admins/1,
|
||||||
|
@ -47,6 +47,8 @@ conf(["load", Path]) ->
|
||||||
load_config(Path);
|
load_config(Path);
|
||||||
conf(["cluster_sync" | Args]) ->
|
conf(["cluster_sync" | Args]) ->
|
||||||
admins(Args);
|
admins(Args);
|
||||||
|
conf(["reload"]) ->
|
||||||
|
emqx_conf:reload_etc_conf_on_local_node();
|
||||||
conf(_) ->
|
conf(_) ->
|
||||||
emqx_ctl:usage(usage_conf() ++ usage_sync()).
|
emqx_ctl:usage(usage_conf() ++ usage_sync()).
|
||||||
|
|
||||||
|
@ -175,6 +177,9 @@ get_config(Key) ->
|
||||||
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
|
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
|
||||||
load_config(Path) ->
|
load_config(Path) ->
|
||||||
case hocon:files([Path]) of
|
case hocon:files([Path]) of
|
||||||
|
{ok, RawConf} when RawConf =:= #{} ->
|
||||||
|
emqx_ctl:warning("load ~ts is empty~n", [Path]),
|
||||||
|
{error, empty_hocon_file};
|
||||||
{ok, RawConf} ->
|
{ok, RawConf} ->
|
||||||
case check_config_keys(RawConf) of
|
case check_config_keys(RawConf) of
|
||||||
ok ->
|
ok ->
|
||||||
|
@ -200,8 +205,8 @@ update_config(Key, Value) ->
|
||||||
end.
|
end.
|
||||||
check_config_keys(Conf) ->
|
check_config_keys(Conf) ->
|
||||||
Keys = maps:keys(Conf),
|
Keys = maps:keys(Conf),
|
||||||
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS],
|
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
|
||||||
case ReadOnlyKeys -- Keys of
|
case ReadOnlyKeys -- Keys of
|
||||||
ReadOnlyKeys -> ok;
|
ReadOnlyKeys -> ok;
|
||||||
_ -> {error, "update_read_only_keys_prohibited"}
|
_ -> {error, "update_readonly_keys_prohibited"}
|
||||||
end.
|
end.
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
%%--------------------------------------------------------------------
|
||||||
|
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||||
|
%%
|
||||||
|
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
%% you may not use this file except in compliance with the License.
|
||||||
|
%% You may obtain a copy of the License at
|
||||||
|
%%
|
||||||
|
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
%%
|
||||||
|
%% Unless required by applicable law or agreed to in writing, software
|
||||||
|
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
%% See the License for the specific language governing permissions and
|
||||||
|
%% limitations under the License.
|
||||||
|
%%--------------------------------------------------------------------
|
||||||
|
-module(emqx_conf_cli_SUITE).
|
||||||
|
|
||||||
|
-compile(nowarn_export_all).
|
||||||
|
-compile(export_all).
|
||||||
|
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
|
||||||
|
-include("emqx_conf.hrl").
|
||||||
|
-import(emqx_config_SUITE, [prepare_conf_file/3]).
|
||||||
|
|
||||||
|
all() ->
|
||||||
|
emqx_common_test_helpers:all(?MODULE).
|
||||||
|
|
||||||
|
init_per_suite(Config) ->
|
||||||
|
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]),
|
||||||
|
Config.
|
||||||
|
|
||||||
|
end_per_suite(_Config) ->
|
||||||
|
emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
|
||||||
|
|
||||||
|
t_load_config(Config) ->
|
||||||
|
Authz = authorization,
|
||||||
|
Conf = emqx_conf:get_raw([Authz]),
|
||||||
|
%% set sources to []
|
||||||
|
ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
|
||||||
|
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
|
||||||
|
ok = emqx_conf_cli:conf(["load", ConfFile0]),
|
||||||
|
?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])),
|
||||||
|
%% remove sources, it will reset to default file source.
|
||||||
|
ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
|
||||||
|
ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
|
||||||
|
ok = emqx_conf_cli:conf(["load", ConfFile1]),
|
||||||
|
Default = [emqx_authz_schema:default_authz()],
|
||||||
|
?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])),
|
||||||
|
%% reset
|
||||||
|
ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
|
||||||
|
ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
|
||||||
|
ok = emqx_conf_cli:conf(["load", ConfFile2]),
|
||||||
|
?assertEqual(Conf, emqx_conf:get_raw([Authz])),
|
||||||
|
?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
t_load_readonly(Config) ->
|
||||||
|
Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])},
|
||||||
|
lists:foreach(
|
||||||
|
fun(Key) ->
|
||||||
|
Conf = emqx_conf:get_raw([Key]),
|
||||||
|
ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}),
|
||||||
|
ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
|
||||||
|
?assertEqual(
|
||||||
|
{error, "update_readonly_keys_prohibited"},
|
||||||
|
emqx_conf_cli:conf(["load", ConfFile0])
|
||||||
|
)
|
||||||
|
end,
|
||||||
|
?READONLY_KEYS
|
||||||
|
),
|
||||||
|
ok.
|
Loading…
Reference in New Issue