diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 086447c8c..223d5aa50 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -155,9 +155,12 @@ get_alarms(deactivated) -> pre_config_update(#{<<"validity_period">> := Period0} = NewConf, OldConf) -> ?MODULE ! {update_timer, hocon_postprocess:duration(Period0)}, - maps:merge(OldConf, NewConf); + merge(OldConf, NewConf); pre_config_update(NewConf, OldConf) -> - maps:merge(OldConf, NewConf). + merge(OldConf, NewConf). + +merge(undefined, New) -> New; +merge(Old, New) -> maps:merge(Old, New). format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) -> Now = erlang:system_time(microsecond), diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 3a2e2b123..671caa822 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -23,6 +23,7 @@ , stop/1 , get_description/0 , get_release/0 + , set_init_config_load_done/0 ]). -include("emqx.hrl"). @@ -43,8 +44,8 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> - emqx_config:load(), - set_backtrace_depth(), + ok = maybe_load_config(), + ok = set_backtrace_depth(), print_otp_version_warning(), print_banner(), %% Load application first for ekka_mnesia scanner @@ -71,6 +72,22 @@ prep_stop(_State) -> stop(_State) -> ok. +%% @doc Call this function to make emqx boot without loading config, +%% in case we want to delegate the config load to a higher level app +%% which manages emqx app. +set_init_config_load_done() -> + application:set_env(emqx, init_config_load_done, true). + +maybe_load_config() -> + case application:get_env(emqx, init_config_load_done, false) of + true -> + ok; + false -> + %% the app env 'config_files' should be set before emqx get started. + ConfFiles = application:get_env(emqx, config_files, []), + emqx_config:init_load(emqx_schema, ConfFiles) + end. + set_backtrace_depth() -> Depth = emqx_config:get([node, backtrace_depth]), _ = erlang:system_flag(backtrace_depth, Depth), diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f7d97eb2c..eaea5050b 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -15,9 +15,9 @@ %%-------------------------------------------------------------------- -module(emqx_config). --compile({no_auto_import, [get/0, get/1]}). +-compile({no_auto_import, [get/0, get/1, put/2]}). --export([ load/0 +-export([ init_load/2 , read_override_conf/0 , save_configs/2 , save_to_app_env/1 @@ -27,8 +27,10 @@ , to_plainmap/1 ]). --export([ get/0 - , get/1 +-export([get_root/1, + get_root_raw/1]). + +-export([ get/1 , get/2 , find/1 , put/1 @@ -52,15 +54,14 @@ ]). %% raw configs is the config that is now parsed and tranlated by hocon schema --export([ get_raw/0 - , get_raw/1 +-export([ get_raw/1 , get_raw/2 , put_raw/1 , put_raw/2 ]). --define(CONF, ?MODULE). --define(RAW_CONF, {?MODULE, raw}). +-define(CONF, fun(ROOT) -> {?MODULE, bin(ROOT)} end). +-define(RAW_CONF, fun(ROOT) -> {?MODULE, raw, bin(ROOT)} end). -define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]). -define(LISTENER_CONF_PATH(ZONE, LISTENER, PATH), [zones, ZONE, listeners, LISTENER | PATH]). @@ -69,22 +70,28 @@ -type raw_config() :: #{binary() => term()} | undefined. -type config() :: #{atom() => term()} | undefined. --spec get() -> map(). -get() -> - persistent_term:get(?CONF, #{}). +%% @doc For the given path, get root value enclosed in a single-key map. +-spec get_root(emqx_map_lib:config_key_path()) -> map(). +get_root([RootName | _]) -> + #{RootName => do_get(?CONF, [RootName])}. +%% @doc For the given path, get raw root value enclosed in a single-key map. +%% key is ensured to be binary. +get_root_raw([RootName | _]) -> + #{bin(RootName) => do_get(?RAW_CONF, [RootName])}. + +%% @doc Get a config value for the given path. +%% The path should at least include root config name. -spec get(emqx_map_lib:config_key_path()) -> term(). -get(KeyPath) -> - emqx_map_lib:deep_get(KeyPath, get()). +get(KeyPath) -> do_get(?CONF, KeyPath). -spec get(emqx_map_lib:config_key_path(), term()) -> term(). -get(KeyPath, Default) -> - emqx_map_lib:deep_get(KeyPath, get(), Default). +get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). -spec find(emqx_map_lib:config_key_path()) -> {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. find(KeyPath) -> - emqx_map_lib:deep_find(KeyPath, get()). + emqx_map_lib:deep_find(KeyPath, get_root(KeyPath)). -spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). get_zone_conf(Zone, KeyPath) -> @@ -122,11 +129,12 @@ find_listener_conf(Zone, Listener, KeyPath) -> -spec put(map()) -> ok. put(Config) -> - persistent_term:put(?CONF, Config). + maps:fold(fun(RootName, RootValue, _) -> + ?MODULE:put([RootName], RootValue) + end, [], Config). -spec put(emqx_map_lib:config_key_path(), term()) -> ok. -put(KeyPath, Config) -> - put(emqx_map_lib:deep_put(KeyPath, get(), Config)). +put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config). -spec update(emqx_map_lib:config_key_path(), update_request()) -> ok | {error, term()}. @@ -137,37 +145,35 @@ update(ConfKeyPath, UpdateReq) -> remove(ConfKeyPath) -> emqx_config_handler:remove_config(ConfKeyPath). --spec get_raw() -> map(). -get_raw() -> - persistent_term:get(?RAW_CONF, #{}). - -spec get_raw(emqx_map_lib:config_key_path()) -> term(). -get_raw(KeyPath) -> - emqx_map_lib:deep_get(KeyPath, get_raw()). +get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath). -spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). -get_raw(KeyPath, Default) -> - emqx_map_lib:deep_get(KeyPath, get_raw(), Default). +get_raw(KeyPath, Default) -> do_get(?RAW_CONF, KeyPath, Default). -spec put_raw(map()) -> ok. put_raw(Config) -> - persistent_term:put(?RAW_CONF, Config). + maps:fold(fun(RootName, RootV, _) -> + ?MODULE:put_raw([RootName], RootV) + end, [], hocon_schema:get_value([], Config)). -spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. -put_raw(KeyPath, Config) -> - put_raw(emqx_map_lib:deep_put(KeyPath, get_raw(), Config)). +put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config). %%============================================================================ %% Load/Update configs From/To files %%============================================================================ -load() -> - %% the app env 'config_files' should be set before emqx get started. - ConfFiles = application:get_env(emqx, config_files, []), + +%% @doc Initial load of the given config files. +%% NOTE: The order of the files is significant, configs from files orderd +%% in the rear of the list overrides prior values. +-spec init_load(module(), [string()]) -> ok. +init_load(SchemaModule, ConfFiles) -> RawRichConf = lists:foldl(fun(ConfFile, Acc) -> Raw = load_hocon_file(ConfFile, richmap), emqx_map_lib:deep_merge(Acc, Raw) end, #{}, ConfFiles), - {_MappedEnvs, RichConf} = hocon_schema:map_translate(emqx_schema, RawRichConf, #{}), + {_MappedEnvs, RichConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, #{}), ok = save_to_emqx_config(to_plainmap(RichConf), to_plainmap(RawRichConf)). -spec read_override_conf() -> raw_config(). @@ -180,7 +186,8 @@ save_configs(RawConf, OverrideConf) -> %% We may need also support hot config update for the apps that use application envs. %% If that is the case uncomment the following line to update the configs to application env %save_to_app_env(_MappedEnvs), - save_to_emqx_config(to_plainmap(RichConf), RawConf), + Conf = maps:with(maps:keys(RawConf), to_plainmap(RichConf)), + save_to_emqx_config(Conf, RawConf), save_to_override_conf(OverrideConf). -spec save_to_app_env([tuple()]) -> ok. @@ -191,8 +198,8 @@ save_to_app_env(AppEnvs) -> -spec save_to_emqx_config(config(), raw_config()) -> ok. save_to_emqx_config(Conf, RawConf) -> - emqx_config:put(emqx_map_lib:unsafe_atom_key_map(Conf)), - emqx_config:put_raw(RawConf). + ?MODULE:put(emqx_map_lib:unsafe_atom_key_map(Conf)), + ?MODULE:put_raw(RawConf). -spec save_to_override_conf(raw_config()) -> ok | {error, term()}. save_to_override_conf(RawConf) -> @@ -214,7 +221,7 @@ load_hocon_file(FileName, LoadType) -> end. emqx_override_conf_name() -> - filename:join([emqx_config:get([node, data_dir]), "emqx_override.conf"]). + filename:join([?MODULE:get([node, data_dir]), "emqx_override.conf"]). to_richmap(Map) -> {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}), @@ -222,3 +229,19 @@ to_richmap(Map) -> to_plainmap(RichMap) -> hocon_schema:richmap_to_map(RichMap). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). + +do_get(PtKey, [RootName | KeyPath]) -> + RootV = persistent_term:get(PtKey(RootName), #{}), + emqx_map_lib:deep_get(KeyPath, RootV). + +do_get(PtKey, [RootName | KeyPath], Default) -> + RootV = persistent_term:get(PtKey(RootName), #{}), + emqx_map_lib:deep_get(KeyPath, RootV, Default). + +do_put(PtKey, [RootName | KeyPath], DeepValue) -> + OldValue = do_get(PtKey, [RootName]), + NewValue = emqx_map_lib:deep_put(KeyPath, OldValue, DeepValue), + persistent_term:put(PtKey(RootName), NewValue). diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 7b70b5382..5c021b9ca 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -86,12 +86,12 @@ handle_call({add_child, ConfKeyPath, HandlerName}, _From, handle_call({update_config, ConfKeyPath, UpdateReq}, _From, #{handlers := Handlers} = State) -> - OldConf = emqx_config:get(), - OldRawConf = emqx_config:get_raw(), + OldConf = emqx_config:get_root(ConfKeyPath), + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), try NewRawConf = do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq), - OverrideConf = update_override_config(ConfKeyPath, NewRawConf), + OverrideConf = update_override_config(NewRawConf), Result = emqx_config:save_configs(NewRawConf, OverrideConf), - do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get()), + do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get_root(ConfKeyPath)), {reply, Result, State} catch Error : Reason : ST -> @@ -100,13 +100,13 @@ handle_call({update_config, ConfKeyPath, UpdateReq}, _From, end; handle_call({remove_config, ConfKeyPath}, _From, #{handlers := Handlers} = State) -> - OldConf = emqx_config:get(), - OldRawConf = emqx_config:get_raw(), + OldConf = emqx_config:get_root(ConfKeyPath), + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), try NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()), Result = emqx_config:save_configs(NewRawConf, OverrideConf), - do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get()), + do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get_root(ConfKeyPath)), {reply, Result, State} catch Error : Reason : ST -> @@ -176,18 +176,11 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf) merge_to_old_config(UpdateReq, _RawConf) -> UpdateReq. -update_override_config(ConfKeyPath, RawConf) -> - %% We don't save the entire config to emqx_override.conf, but only the part - %% specified by the ConfKeyPath - PartialConf = maps:with(root_keys(ConfKeyPath), RawConf), +update_override_config(RawConf) -> OldConf = emqx_config:read_override_conf(), - maps:merge(OldConf, PartialConf). - -root_keys([]) -> []; -root_keys([RootKey | _]) -> [bin(RootKey)]. + maps:merge(OldConf, RawConf). bin_path(ConfKeyPath) -> [bin(Key) || Key <- ConfKeyPath]. -bin(A) when is_atom(A) -> list_to_binary(atom_to_list(A)); -bin(B) when is_binary(B) -> B; -bin(S) when is_list(S) -> list_to_binary(S). +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index faad35c3d..8c7334cbe 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -306,7 +306,7 @@ merge_zone_and_listener_confs(ZoneConf, ListenerConf) -> apply_on_listener(ListenerId, Do) -> {ZoneName, ListenerName} = decode_listener_id(ListenerId), - case emqx_config:find([zones, ZoneName, listeners, ListenerName]) of - {not_found, _, _} -> error({not_found, ListenerId}); + case emqx_config:find_listener_conf(ZoneName, ListenerName, []) of + {not_found, _, _} -> error({listener_config_not_found, ZoneName, ListenerName}); {ok, Conf} -> Do(ZoneName, ListenerName, Conf) end. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8ccc2971d..3fd060d9f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -594,7 +594,7 @@ tr_logger(Conf) -> filters => Filters, filesync_repeat_interval => no_repeat }} - || {HandlerName, SubConf} <- maps:to_list(conf_get("log.file_handlers", Conf))], + || {HandlerName, SubConf} <- maps:to_list(conf_get("log.file_handlers", Conf, #{}))], [{handler, default, undefined}] ++ ConsoleHandler ++ FileHandlers. diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 1f7ea3113..9614822ba 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -27,7 +27,6 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> emqx_ct_helpers:start_apps([]), - ct:pal("------------config: ~p", [emqx_config:get()]), Config. end_per_suite(_Config) -> diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index 05696fc69..c01420f49 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -25,7 +25,7 @@ all() -> emqx_ct:all(?MODULE). t_check_pub(_) -> - OldConf = emqx_config:get(), + OldConf = emqx_config:get([zones]), emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], ?QOS_1), emqx_config:put_zone_conf(default, [mqtt, retain_available], false), timer:sleep(50), @@ -36,10 +36,10 @@ t_check_pub(_) -> PubFlags2 = #{qos => ?QOS_1, retain => true}, ?assertEqual({error, ?RC_RETAIN_NOT_SUPPORTED}, emqx_mqtt_caps:check_pub(default, PubFlags2)), - emqx_config:put(OldConf). + emqx_config:put([zones], OldConf). t_check_sub(_) -> - OldConf = emqx_config:get(), + OldConf = emqx_config:get([zones]), SubOpts = #{rh => 0, rap => 0, nl => 0, @@ -57,4 +57,4 @@ t_check_sub(_) -> emqx_mqtt_caps:check_sub(default, <<"+/#">>, SubOpts)), ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, emqx_mqtt_caps:check_sub(default, <<"topic">>, SubOpts#{share => true})), - emqx_config:put(OldConf). + emqx_config:put([zones], OldConf).