feat(emqx_config): allow loading configs from binary

This will be helpful for tests

apps/emqx/src/emqx_config.erl

@@ -19,12 +19,10 @@
 
 -export([ init_load/2
         , read_override_conf/0
-        , save_configs/2
+        , save_configs/3
         , save_to_app_env/1
         , save_to_emqx_config/2
         , save_to_override_conf/1
-        , to_richmap/1
-        , to_plainmap/1
         ]).
 
 -export([get_root/1,
@@ -50,7 +48,9 @@
         ]).
 
 -export([ update/2
+        , update/3
         , remove/1
+        , remove/2
         ]).
 
 %% raw configs is the config that is now parsed and tranlated by hocon schema
@@ -139,11 +139,19 @@ put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config).
 -spec update(emqx_map_lib:config_key_path(), update_request()) ->
     ok | {error, term()}.
 update(ConfKeyPath, UpdateReq) ->
-    emqx_config_handler:update_config(ConfKeyPath, UpdateReq).
+    update(emqx_schema, ConfKeyPath, UpdateReq).
+
+-spec update(module(), emqx_map_lib:config_key_path(), update_request()) ->
+    ok | {error, term()}.
+update(SchemaModule, ConfKeyPath, UpdateReq) ->
+    emqx_config_handler:update_config(SchemaModule, ConfKeyPath, UpdateReq).
 
 -spec remove(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
 remove(ConfKeyPath) ->
-    emqx_config_handler:remove_config(ConfKeyPath).
+    remove(emqx_schema, ConfKeyPath).
+
+remove(SchemaModule, ConfKeyPath) ->
+    emqx_config_handler:remove_config(SchemaModule, ConfKeyPath).
 
 -spec get_raw(emqx_map_lib:config_key_path()) -> term().
 get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath).
@@ -167,27 +175,51 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config).
 %% @doc Initial load of the given config files.
 %% NOTE: The order of the files is significant, configs from files orderd
 %% in the rear of the list overrides prior values.
--spec init_load(module(), [string()]) -> ok.
+-spec init_load(module(), [string()] | binary() | hocon:config()) -> ok.
-init_load(SchemaModule, ConfFiles) ->
+init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) ->
-    RawRichConf = lists:foldl(fun(ConfFile, Acc) ->
+    ParseOptions = #{format => richmap},
-        Raw = load_hocon_file(ConfFile, richmap),
+    Parser = case is_binary(Conf) of
-        emqx_map_lib:deep_merge(Acc, Raw)
+              true -> fun hocon:binary/2;
-    end, #{}, ConfFiles),
+              false -> fun hocon:files/2
-    {_MappedEnvs, RichConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, #{}),
+             end,
-    ok = save_to_emqx_config(to_plainmap(RichConf), to_plainmap(RawRichConf)).
+    case Parser(Conf, ParseOptions) of
+        {ok, RawRichConf} ->
+            init_load(SchemaModule, RawRichConf);
+        {error, Reason} ->
+            logger:error(#{msg => failed_to_load_hocon_conf,
+                           reason => Reason
+                          }),
+            error(failed_to_load_hocon_conf)
+    end;
+init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) ->
+    RawConf =  hocon_schema:richmap_to_map(RawRichConf),
+    %% check with richmap for line numbers in error reports (future enhancement)
+    Opts = #{return_plain => true,
+             nullable => true
+            },
+    %% this call throws exception in case of check failure
+    {_MappedEnvs, CheckedConf} =
+        hocon_schema:map_translate(SchemaModule, RawRichConf, Opts),
+    ok = save_to_emqx_config(CheckedConf, RawConf).
 
 -spec read_override_conf() -> raw_config().
 read_override_conf() ->
     load_hocon_file(emqx_override_conf_name(), map).
 
--spec save_configs(raw_config(), raw_config()) -> ok | {error, term()}.
+-spec save_configs(module(), raw_config(), raw_config()) -> ok | {error, term()}.
-save_configs(RawConf, OverrideConf) ->
+save_configs(SchemaModule, RawConf, OverrideConf) ->
-    {_MappedEnvs, RichConf} = hocon_schema:map_translate(emqx_schema, to_richmap(RawConf), #{}),
+    Opts = #{return_plain => true,
+             nullable => true,
+             is_richmap => false
+            },
+    {_MappedEnvs, CheckedConf} =
+        hocon_schema:map_translate(SchemaModule, RawConf, Opts),
     %% We may need also support hot config update for the apps that use application envs.
     %% If that is the case uncomment the following line to update the configs to application env
     %save_to_app_env(_MappedEnvs),
-    Conf = maps:with(maps:keys(RawConf), to_plainmap(RichConf)),
+    Conf = maps:with(maps:keys(RawConf), CheckedConf),
     save_to_emqx_config(Conf, RawConf),
+    %% TODO: merge RawConf to OverrideConf can be done here
     save_to_override_conf(OverrideConf).
 
 -spec save_to_app_env([tuple()]) -> ok.
@@ -223,13 +255,6 @@ load_hocon_file(FileName, LoadType) ->
 emqx_override_conf_name() ->
     filename:join([?MODULE:get([node, data_dir]), "emqx_override.conf"]).
 
-to_richmap(Map) ->
-    {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}),
-    RichMap.
-
-to_plainmap(RichMap) ->
-    hocon_schema:richmap_to_map(RichMap).
-
 bin(Bin) when is_binary(Bin) -> Bin;
 bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
 

apps/emqx/src/emqx_config_handler.erl

@@ -24,8 +24,8 @@
 %% API functions
 -export([ start_link/0
         , add_handler/2
-        , update_config/2
+        , update_config/3
-        , remove_config/1
+        , remove_config/2
         , merge_to_old_config/2
         ]).
 
@@ -59,15 +59,15 @@
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []).
 
--spec update_config(emqx_config:config_key_path(), emqx_config:update_request()) ->
+-spec update_config(module(), emqx_config:config_key_path(), emqx_config:update_request()) ->
     ok | {error, term()}.
-update_config(ConfKeyPath, UpdateReq) ->
+update_config(SchemaModule, ConfKeyPath, UpdateReq) ->
-    gen_server:call(?MODULE, {update_config, ConfKeyPath, UpdateReq}).
+    gen_server:call(?MODULE, {update_config, SchemaModule, ConfKeyPath, UpdateReq}).
 
--spec remove_config(emqx_config:config_key_path()) ->
+-spec remove_config(module(), emqx_config:config_key_path()) ->
     ok | {error, term()}.
-remove_config(ConfKeyPath) ->
+remove_config(SchemaModule, ConfKeyPath) ->
-    gen_server:call(?MODULE, {remove_config, ConfKeyPath}).
+    gen_server:call(?MODULE, {remove_config, SchemaModule, ConfKeyPath}).
 
 -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok.
 add_handler(ConfKeyPath, HandlerName) ->
@@ -84,13 +84,13 @@ handle_call({add_child, ConfKeyPath, HandlerName}, _From,
     {reply, ok, State#{handlers =>
         emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}};
 
-handle_call({update_config, ConfKeyPath, UpdateReq}, _From,
+handle_call({update_config, SchemaModule, ConfKeyPath, UpdateReq}, _From,
             #{handlers := Handlers} = State) ->
     OldConf = emqx_config:get_root(ConfKeyPath),
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
     try NewRawConf = do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq),
         OverrideConf = update_override_config(NewRawConf),
-        Result = emqx_config:save_configs(NewRawConf, OverrideConf),
+        Result = emqx_config:save_configs(SchemaModule, NewRawConf, OverrideConf),
         do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get_root(ConfKeyPath)),
         {reply, Result, State}
     catch
@@ -99,13 +99,13 @@ handle_call({update_config, ConfKeyPath, UpdateReq}, _From,
             {reply, {error, Reason}, State}
     end;
 
-handle_call({remove_config, ConfKeyPath}, _From, #{handlers := Handlers} = State) ->
+handle_call({remove_config, SchemaModule, ConfKeyPath}, _From, #{handlers := Handlers} = State) ->
     OldConf = emqx_config:get_root(ConfKeyPath),
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
     BinKeyPath = bin_path(ConfKeyPath),
     try NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
         OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()),
-        Result = emqx_config:save_configs(NewRawConf, OverrideConf),
+        Result = emqx_config:save_configs(SchemaModule, NewRawConf, OverrideConf),
         do_post_config_update(ConfKeyPath, Handlers, OldConf, emqx_config:get_root(ConfKeyPath)),
         {reply, Result, State}
     catch

rebar.config

@@ -60,7 +60,7 @@
     , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
     , {getopt, "1.0.1"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.4"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.5"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.3.0"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}}
     ]}.