Add some APIs for emqx_config (#5095)

apps/emqx/src/emqx_config.erl

@@ -18,43 +18,95 @@
 -compile({no_auto_import, [get/0, get/1]}).
 
 -export([ get/0
+        , get/1
         , get/2
         , put/1
         , put/2
+        ]).
+
+-export([ update_config/2
+        ]).
+
+%% raw configs is the config that is now parsed and tranlated by hocon schema
+-export([ get_raw/0
+        , get_raw/1
+        , get_raw/2
+        , put_raw/1
+        , put_raw/2
+        ]).
+
+-export([ deep_get/2
         , deep_get/3
         , deep_put/3
         , safe_atom_key_map/1
         , unsafe_atom_key_map/1
         ]).
 
--spec get() -> term().
+-define(CONF, ?MODULE).
-get() ->
+-define(RAW_CONF, {?MODULE, raw}).
-    persistent_term:get(?MODULE, #{}).
 
--spec get([atom()], term()) -> term().
+-export_type([update_request/0, raw_config/0, config_key/0, config_key_path/0]).
+-type update_request() :: term().
+-type raw_config() :: hocon:config() | undefined.
+-type config_key() :: atom() | binary().
+-type config_key_path() :: [config_key()].
+
+-spec get() -> map().
+get() ->
+    persistent_term:get(?CONF, #{}).
+
+-spec get(config_key_path()) -> term().
+get(KeyPath) ->
+    deep_get(KeyPath, get()).
+
+-spec get(config_key_path(), term()) -> term().
 get(KeyPath, Default) ->
     deep_get(KeyPath, get(), Default).
 
--spec deep_get([atom()], map(), term()) -> term().
+-spec put(map()) -> ok.
-deep_get([], Map, _Default) ->
-    Map;
-deep_get([Key | KeyPath], Map, Default) when is_map(Map) ->
-    case maps:find(Key, Map) of
-        {ok, SubMap} -> deep_get(KeyPath, SubMap, Default);
-        error -> Default
-    end;
-deep_get([_Key | _KeyPath], _Map, Default) ->
-    Default.
-
--spec put(term()) -> ok.
 put(Config) ->
-    persistent_term:put(?MODULE, Config).
+    persistent_term:put(?CONF, Config).
 
--spec put([atom()], term()) -> ok.
+-spec put(config_key_path(), term()) -> ok.
 put(KeyPath, Config) ->
     put(deep_put(KeyPath, get(), Config)).
 
--spec deep_put([atom()], map(), term()) -> ok.
+-spec update_config(config_key_path(), update_request()) ->
+    ok | {error, term()}.
+update_config(ConfKeyPath, UpdateReq) ->
+    emqx_config_handler:update_config(ConfKeyPath, UpdateReq, get_raw()).
+
+-spec get_raw() -> map().
+get_raw() ->
+    persistent_term:get(?RAW_CONF, #{}).
+
+-spec get_raw(config_key_path()) -> term().
+get_raw(KeyPath) ->
+    deep_get(KeyPath, get_raw()).
+
+-spec get_raw(config_key_path(), term()) -> term().
+get_raw(KeyPath, Default) ->
+    deep_get(KeyPath, get_raw(), Default).
+
+-spec put_raw(map()) -> ok.
+put_raw(Config) ->
+    persistent_term:put(?RAW_CONF, Config).
+
+-spec put_raw(config_key_path(), term()) -> ok.
+put_raw(KeyPath, Config) ->
+    put_raw(deep_put(KeyPath, get_raw(), Config)).
+
+%%-----------------------------------------------------------------
+-spec deep_get(config_key_path(), map()) -> term().
+deep_get(ConfKeyPath, Map) ->
+    do_deep_get(ConfKeyPath, Map, fun(KeyPath, Data) ->
+        error({not_found, KeyPath, Data}) end).
+
+-spec deep_get(config_key_path(), map(), term()) -> term().
+deep_get(ConfKeyPath, Map, Default) ->
+    do_deep_get(ConfKeyPath, Map, fun(_, _) -> Default end).
+
+-spec deep_put(config_key_path(), map(), term()) -> map().
 deep_put([], Map, Config) when is_map(Map) ->
     Config;
 deep_put([Key | KeyPath], Map, Config) ->
@@ -67,6 +119,19 @@ unsafe_atom_key_map(Map) ->
 safe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).
 
+%%---------------------------------------------------------------------------
+
+-spec do_deep_get(config_key_path(), map(), fun((config_key(), term()) -> any())) -> term().
+do_deep_get([], Map, _) ->
+    Map;
+do_deep_get([Key | KeyPath], Map, OnNotFound) when is_map(Map) ->
+    case maps:find(Key, Map) of
+        {ok, SubMap} -> do_deep_get(KeyPath, SubMap, OnNotFound);
+        error -> OnNotFound(Key, Map)
+    end;
+do_deep_get([Key | _KeyPath], Data, OnNotFound) ->
+    OnNotFound(Key, Data).
+
 covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) ->
     maps:fold(
         fun(K, V, Acc) when is_binary(K) ->

apps/emqx/src/emqx_config_handler.erl

@@ -24,8 +24,7 @@
 %% API functions
 -export([ start_link/0
         , add_handler/2
-        , update_config/2
+        , update_config/3
-        , get_raw_config/0
         , merge_to_old_config/2
         ]).
 
@@ -43,60 +42,49 @@
 
 -define(MOD, {mod}).
 
--type update_request() :: term().
--type raw_config() :: hocon:config() | undefined.
--type config_key() :: atom().
 -type handler_name() :: module().
--type config_key_path() :: [atom()].
+-type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}.
--type handlers() :: #{config_key() => handlers(), ?MOD => handler_name()}.
 
 -optional_callbacks([handle_update_config/2]).
 
--callback handle_update_config(update_request(), raw_config()) -> update_request().
+-callback handle_update_config(emqx_config:update_request(), emqx_config:raw_config()) ->
+    emqx_config:update_request().
 
 -type state() :: #{
     handlers := handlers(),
-    raw_config := raw_config(),
     atom() => term()
 }.
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []).
 
--spec update_config(config_key_path(), update_request()) -> ok | {error, term()}.
+-spec update_config(emqx_config:config_key_path(), emqx_config:update_request(),
-update_config(ConfKeyPath, UpdateReq) ->
+        emqx_config:raw_config()) ->
-    gen_server:call(?MODULE, {update_config, ConfKeyPath, UpdateReq}).
+    ok | {error, term()}.
+update_config(ConfKeyPath, UpdateReq, RawConfig) ->
+    gen_server:call(?MODULE, {update_config, ConfKeyPath, UpdateReq, RawConfig}).
 
--spec add_handler(config_key_path(), handler_name()) -> ok.
+-spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok.
 add_handler(ConfKeyPath, HandlerName) ->
     gen_server:call(?MODULE, {add_child, ConfKeyPath, HandlerName}).
 
--spec get_raw_config() -> raw_config().
-get_raw_config() ->
-    gen_server:call(?MODULE, get_raw_config).
-
 %%============================================================================
 
 -spec init(term()) -> {ok, state()}.
 init(_) ->
     {ok, RawConf} = hocon:load(emqx_conf_name(), #{format => richmap}),
     {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, RawConf, #{}),
-    ok = save_config_to_emqx_config(hocon_schema:richmap_to_map(Conf)),
+    ok = save_config_to_emqx(to_plainmap(Conf), to_plainmap(RawConf)),
-    {ok, #{raw_config => hocon_schema:richmap_to_map(RawConf),
+    {ok, #{handlers => #{?MOD => ?MODULE}}}.
-           handlers => #{?MOD => ?MODULE}}}.
-
-handle_call(get_raw_config, _From, State = #{raw_config := RawConf}) ->
-    {reply, RawConf, State};
-
 handle_call({add_child, ConfKeyPath, HandlerName}, _From,
             State = #{handlers := Handlers}) ->
     {reply, ok, State#{handlers =>
         emqx_config:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}};
 
-handle_call({update_config, ConfKeyPath, UpdateReq}, _From,
+handle_call({update_config, ConfKeyPath, UpdateReq, RawConf}, _From,
-        #{raw_config := RawConf, handlers := Handlers} = State) ->
+            #{handlers := Handlers} = State) ->
     try {RootKeys, Conf} = do_update_config(ConfKeyPath, Handlers, RawConf, UpdateReq),
-        {reply, save_configs(RootKeys, Conf), State#{raw_config => Conf}}
+        {reply, save_configs(RootKeys, Conf), State}
     catch
         throw: Reason ->
             {reply, {error, Reason}, State};
@@ -158,21 +146,22 @@ merge_to_old_config(UpdateReq, RawConf) ->
     maps:merge(RawConf, UpdateReq).
 
 %%============================================================================
-save_configs(RootKeys, Conf0) ->
+save_configs(RootKeys, RawConf) ->
-    {_MappedEnvs, Conf1} = hocon_schema:map_translate(emqx_schema, to_richmap(Conf0), #{}),
+    {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, to_richmap(RawConf), #{}),
-    %save_config_to_app_env(MappedEnvs),
-    save_config_to_emqx_config(hocon_schema:richmap_to_map(Conf1)),
-    save_config_to_disk(RootKeys, Conf0).
-
     %% We may need also support hot config update for the apps that use application envs.
-%% If so uncomment the following lines to update the configs to application env
+    %% If so uncomment the following line to update the configs to application env
+    %save_config_to_app_env(_MappedEnvs),
+    save_config_to_emqx(to_plainmap(Conf), RawConf),
+    save_config_to_disk(RootKeys, RawConf).
+
 % save_config_to_app_env(MappedEnvs) ->
 %     lists:foreach(fun({AppName, Envs}) ->
 %             [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs]
 %         end, MappedEnvs).
 
-save_config_to_emqx_config(Conf) ->
+save_config_to_emqx(Conf, RawConf) ->
-    emqx_config:put(emqx_config:unsafe_atom_key_map(Conf)).
+    emqx_config:put(emqx_config:unsafe_atom_key_map(Conf)),
+    emqx_config:put_raw(RawConf).
 
 save_config_to_disk(RootKeys, Conf) ->
     FileName = emqx_override_conf_name(),
@@ -215,6 +204,9 @@ to_richmap(Map) ->
     {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}),
     RichMap.
 
+to_plainmap(RichMap) ->
+    hocon_schema:richmap_to_map(RichMap).
+
 bin(A) when is_atom(A) -> list_to_binary(atom_to_list(A));
 bin(B) when is_binary(B) -> B;
 bin(S) when is_list(S) -> list_to_binary(S).

apps/emqx_data_bridge/src/emqx_data_bridge.erl

@@ -60,4 +60,4 @@ config_key_path() ->
     [emqx_data_bridge, bridges].
 
 update_config(ConfigReq) ->
-    emqx_config_handler:update_config(config_key_path(), ConfigReq).
+    emqx_config:update_config(config_key_path(), ConfigReq).