Merge pull request #6845 from zmstone/refactor-config-3-layers

Refactor config 3 layers
This commit is contained in:
zhongwencool 2022-01-26 11:28:10 +08:00 committed by GitHub
commit 53b707b9f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 98 additions and 67 deletions

View File

@ -21,6 +21,7 @@
-export([ init_load/1
, init_load/2
, read_override_conf/1
, delete_override_conf_files/0
, check_config/2
, fill_defaults/1
, fill_defaults/2
@ -252,40 +253,49 @@ init_load(SchemaMod) ->
%% in the rear of the list overrides prior values.
-spec init_load(module(), [string()] | binary() | hocon:config()) -> ok.
init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
IncDir = include_dirs(),
ParseOptions = #{format => map, include_dirs => IncDir},
Parser = case is_binary(Conf) of
true -> fun hocon:binary/2;
false -> fun hocon:files/2
end,
case Parser(Conf, ParseOptions) of
{ok, RawRichConf} ->
init_load(SchemaMod, RawRichConf);
init_load(SchemaMod, parse_hocon(Conf));
init_load(SchemaMod, RawConf) when is_map(RawConf) ->
ok = save_schema_mod_and_names(SchemaMod),
%% Merge environment varialbe overrides on top
RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
ClusterOverrides = read_override_conf(#{override_to => cluster}),
LocalOverrides = read_override_conf(#{override_to => local}),
Overrides = hocon:deep_merge(ClusterOverrides, LocalOverrides),
RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides),
%% check configs against the schema
{_AppEnvs, CheckedConf} =
check_config(SchemaMod, RawConfWithOverrides , #{}),
RootNames = get_root_names(),
ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf),
maps:with(RootNames, RawConfWithEnvs)).
parse_hocon(Conf) ->
IncDirs = include_dirs(),
case do_parse_hocon(Conf, IncDirs) of
{ok, HoconMap} ->
HoconMap;
{error, Reason} ->
?SLOG(error, #{msg => "failed_to_load_hocon_conf",
reason => Reason,
pwd => file:get_cwd(),
include_dirs => IncDir
include_dirs => IncDirs,
config_file => Conf
}),
error(failed_to_load_hocon_conf)
end;
init_load(SchemaMod, RawConf) when is_map(RawConf) ->
ok = save_schema_mod_and_names(SchemaMod),
%% check configs agains the schema, with environment variables applied on top
{_AppEnvs, CheckedConf} =
check_config(SchemaMod, RawConf, #{apply_override_envs => true}),
%% fill default values for raw config
RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
RootNames = get_root_names(),
ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf),
maps:with(RootNames, RawConfWithEnvs)).
end.
do_parse_hocon(Conf, IncDirs) ->
Opts = #{format => map, include_dirs => IncDirs},
case is_binary(Conf) of
true -> hocon:binary(Conf, Opts);
false -> hocon:files(Conf, Opts)
end.
include_dirs() ->
[filename:join(emqx:data_dir(), "configs")].
merge_envs(SchemaMod, RawConf) ->
Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config
nullable => true, %% TODO: evil, remove, nullable should be declared in schema
Opts = #{nullable => true, %% TODO: evil, remove, nullable should be declared in schema
format => map,
apply_override_envs => true
},
@ -324,6 +334,23 @@ fill_defaults(SchemaMod, RawConf) ->
#{nullable => true, only_fill_defaults => true},
root_names_from_conf(RawConf)).
%% @doc Only for test cleanups.
%% Delete override config files.
-spec delete_override_conf_files() -> ok.
delete_override_conf_files() ->
F1 = override_conf_file(#{override_to => local}),
F2 = override_conf_file(#{override_to => cluster}),
ok = ensure_file_deleted(F1),
ok = ensure_file_deleted(F2).
ensure_file_deleted(F) ->
case file:delete(F) of
ok -> ok;
{error, enoent} -> ok;
{error, Reason} -> error({F, Reason})
end.
-spec read_override_conf(map()) -> raw_config().
read_override_conf(#{} = Opts) ->
File = override_conf_file(Opts),

View File

@ -48,6 +48,7 @@
, not_wait_mqtt_payload/1
, render_config_file/2
, read_schema_configs/2
, load_config/2
]).
-define( CERTS_PATH(CertName), filename:join( [ "etc", "certs", CertName ]) ).
@ -428,3 +429,7 @@ copy_certs(emqx_conf, Dest0) ->
os:cmd( ["cp -rf ", From, "/certs ", Dest, "/"]),
ok;
copy_certs(_, _) -> ok.
load_config(SchemaModule, Config) ->
ok = emqx_config:delete_override_conf_files(),
ok = emqx_config:init_load(SchemaModule, Config).

View File

@ -96,7 +96,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF),
emqx_common_test_helpers:start_apps([?APP]),
Config.
@ -107,7 +107,7 @@ init_per_testcase(_TestCase, Config) ->
Config.
base_conf() ->
emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF).
emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF).
%%--------------------------------------------------------------------
%% Test Cases Bucket Level

View File

@ -96,10 +96,11 @@ fields(file) ->
, {path, #{type => string(),
desc => """
Path to the file which contains the ACL rules.<br>
If the file provisioned before starting EMQ X node, it can be placed anywhere
as long as EMQ X has read access to it.
In case rule set is created from EMQ X dashboard or management HTTP API,
the file will be placed in `certs/authz` sub directory inside EMQ X's `data_dir`,
If the file provisioned before starting EMQ X node,
it can be placed anywhere as long as EMQ X has read access to it.
In case the rule-set is created from EMQ X dashboard or management API,
the file will be placed in `authz` sub directory inside EMQ X's `data_dir`,
and the new rules will override all rules from the old config file.
"""
}}

View File

@ -58,7 +58,7 @@ init_per_suite(Config) ->
application:load(emqx_dashboard),
application:load(?APP),
ok = emqx_config:init_load(emqx_auto_subscribe_schema,
ok = emqx_common_test_helpers:load_config(emqx_auto_subscribe_schema,
<<"auto_subscribe {
topics = [
{

View File

@ -64,7 +64,7 @@ init_per_suite(Config) ->
_ = application:stop(emqx_resource),
_ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_bridge, emqx_dashboard]),
ok = emqx_config:init_load(emqx_bridge_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?CONF_DEFAULT),
Config.
end_per_suite(_Config) ->

View File

@ -97,9 +97,9 @@ init_per_suite(Config) ->
_ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector,
emqx_bridge, emqx_dashboard]),
ok = emqx_config:init_load(emqx_connector_schema, <<"connectors: {}">>),
ok = emqx_config:init_load(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>),
ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_connector_schema, <<"connectors: {}">>),
ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
Config.
end_per_suite(_Config) ->

View File

@ -59,7 +59,7 @@ init_per_suite(Cfg) ->
meck:expect(emqx_alarm, deactivate, 3, ok),
_ = emqx_exhook_demo_svr:start(),
ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([emqx_exhook]),
Cfg.

View File

@ -50,7 +50,7 @@ init_per_suite(Config) ->
meck:expect(emqx_alarm, deactivate, 3, ok),
_ = emqx_exhook_demo_svr:start(),
ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_exhook]),
[Conf] = emqx:get_config([exhook, servers]),
[{template, Conf} | Config].

View File

@ -53,7 +53,7 @@ gateway.coap
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_gateway]),
Config.

View File

@ -54,7 +54,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_gateway]),
Config.

View File

@ -32,7 +32,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([emqx_gateway]),
Conf.

View File

@ -40,7 +40,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
Conf.

View File

@ -54,7 +54,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
Conf.

View File

@ -34,7 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([]),
ok = meck:new(emqx_gateway_metrics, [passthrough, no_history, no_link]),

View File

@ -34,7 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([]),
Conf.

View File

@ -34,7 +34,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:init_load(emqx_gateway_schema, <<"gateway {}">>),
emqx_common_test_helpers:load_config(emqx_gateway_schema, <<"gateway {}">>),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_gateway]),
Conf.

View File

@ -33,7 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([]),
Conf.

View File

@ -34,7 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
%%--------------------------------------------------------------------
init_per_suite(Cfg) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([emqx_gateway]),
Cfg.

View File

@ -173,7 +173,7 @@ end_per_suite(Config) ->
Config.
init_per_testcase(_AllTestCase, Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
{ok, _} = application:ensure_all_started(emqx_gateway),
{ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]),

View File

@ -69,7 +69,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
application:load(emqx_gateway),
emqx_mgmt_api_test_util:init_suite([emqx_conf]),
Config.
@ -81,7 +81,7 @@ end_per_suite(Config) ->
Config.
init_per_testcase(_AllTestCase, Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
{ok, _} = application:ensure_all_started(emqx_gateway),
{ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]),

View File

@ -88,7 +88,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_gateway]),
Config.

View File

@ -49,7 +49,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
%%--------------------------------------------------------------------
init_per_suite(Cfg) ->
ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_gateway]),
Cfg.

View File

@ -38,7 +38,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([emqx_modules]),
ok = emqx_config:init_load(emqx_modules_schema, ?EVENT_MESSAGE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?EVENT_MESSAGE),
Config.
end_per_suite(_Config) ->

View File

@ -29,7 +29,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:init_load(emqx_modules_schema, <<"gateway {}">>),
emqx_common_test_helpers:load_config(emqx_modules_schema, <<"gateway {}">>),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_modules]),
Conf.

View File

@ -112,7 +112,7 @@ t_rewrite_re_error(_Config) ->
ok.
t_list(_Config) ->
ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE),
Expect = [
#{<<"action">> => <<"publish">>,
<<"dest_topic">> => <<"z/y/$1">>,
@ -130,7 +130,7 @@ t_list(_Config) ->
ok.
t_update(_Config) ->
ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE),
Init = emqx_rewrite:list(),
Rules = [#{
<<"source_topic">> => <<"test/#">>,
@ -144,7 +144,7 @@ t_update(_Config) ->
ok.
t_update_disable(_Config) ->
ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE),
?assertEqual(ok, emqx_rewrite:update([])),
timer:sleep(150),
@ -159,7 +159,7 @@ t_update_disable(_Config) ->
ok.
t_update_re_failed(_Config) ->
ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE),
Rules = [#{
<<"source_topic">> => <<"test/#">>,
<<"re">> => <<"*^test/*">>,
@ -188,7 +188,7 @@ receive_publish(Timeout) ->
end.
init() ->
ok = emqx_config:init_load(emqx_modules_schema, ?REWRITE),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE),
ok = emqx_rewrite:enable(),
{ok, C} = emqtt:start_link([{clientid, <<"rewrite_client">>}]),
{ok, _} = emqtt:connect(C),

View File

@ -30,7 +30,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([emqx_modules]),
ok = emqx_config:init_load(emqx_modules_schema, ?TOPIC),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?TOPIC),
Config.
end_per_suite(_Config) ->

View File

@ -57,7 +57,7 @@ init_per_suite(Config) ->
meck:expect(emqx_alarm, activate, 3, ok),
meck:expect(emqx_alarm, deactivate, 3, ok),
ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_retainer_schema, ?BASE_CONF),
emqx_common_test_helpers:start_apps([emqx_retainer]),
Config.

View File

@ -42,7 +42,7 @@ retainer {
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_retainer_schema, ?BASE_CONF),
%% Meck emqtt
ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]),
%% Start Apps

View File

@ -13,7 +13,7 @@ all() ->
init_per_suite(Config) ->
application:load(emqx_conf),
ok = emqx_config:init_load(emqx_rule_engine_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine]),
Config.

View File

@ -40,7 +40,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_config:init_load(emqx_slow_subs_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_slow_subs_schema, ?BASE_CONF),
emqx_common_test_helpers:start_apps([emqx_slow_subs]),
Config.

View File

@ -57,7 +57,7 @@ init_per_suite(Config) ->
meck:expect(emqx_alarm, activate, 3, ok),
meck:expect(emqx_alarm, deactivate, 3, ok),
ok = emqx_config:init_load(emqx_slow_subs_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:load_config(emqx_slow_subs_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_slow_subs]),
{ok, _} = application:ensure_all_started(emqx_authn),
Config.

View File

@ -21,9 +21,7 @@ main(_) ->
false -> Acc
end
end, BaseConf, Cfgs),
ClusterInc = "include \"cluster-override.conf\"\n",
LocalInc = "include \"local-override.conf\"\n",
ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", [Conf, ClusterInc, LocalInc]).
ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf).
get_all_cfgs(Root) ->
Apps = filelib:wildcard("*", Root) -- ["emqx_machine", "emqx_conf"],