feat(emqx_config): add emqx_config:fill_defaults/1

2021-08-11 21:29:03 +08:00 · 2021-08-11 21:29:03 +08:00 · 1c86bd6199
parent a4d29ec8de
commit 1c86bd6199
5 changed files with 147 additions and 31 deletions
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@ -15,7 +15,7 @@
    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}}
    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}}
    , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
    , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
    , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@ -20,14 +20,20 @@
 -export([ init_load/2
        , read_override_conf/0
        , check_config/2
+        , fill_defaults/1
+        , fill_defaults/2
        , save_configs/4
        , save_to_app_env/1
        , save_to_config_map/2
        , save_to_override_conf/1
        ]).

-export([get_root/1,
-         get_root_raw/1]).
+-export([ get_root/1
+        , get_root_raw/1
+        ]).
+
+-export([ get_default_value/1
+        ]).

 -export([ get/1
        , get/2
@ -37,6 +43,12 @@
        , put/2
        ]).

+-export([ save_schema_mod/1
+        , get_schema_mod/0
+        , get_schema_mod/1
+        , get_root_names/0
+        ]).
+
 -export([ get_zone_conf/2
        , get_zone_conf/3
        , put_zone_conf/3
@ -53,6 +65,7 @@
        , update/3
        , remove/1
        , remove/2
+        , reset/1
        ]).

 -export([ get_raw/1
@ -63,6 +76,7 @@

 -define(CONF, conf).
 -define(RAW_CONF, raw_conf).
+-define(PERSIS_MOD_ROOTNAMES, {?MODULE, default_conf}).
 -define(PERSIS_KEY(TYPE, ROOT), {?MODULE, TYPE, ROOT}).
 -define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]).
 -define(LISTENER_CONF_PATH(ZONE, LISTENER, PATH), [zones, ZONE, listeners, LISTENER | PATH]).
@ -170,20 +184,49 @@ put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config).

 -spec update(emqx_map_lib:config_key_path(), update_request()) ->
    ok | {error, term()}.
-update(KeyPath, UpdateReq) ->
-    update(emqx_schema, KeyPath, UpdateReq).
+update([RootName | _] = KeyPath, UpdateReq) ->
+    update(get_schema_mod(RootName), KeyPath, UpdateReq).

 -spec update(module(), emqx_map_lib:config_key_path(), update_request()) ->
    ok | {error, term()}.
-update(SchemaModule, KeyPath, UpdateReq) ->
-    emqx_config_handler:update_config(SchemaModule, KeyPath, {update, UpdateReq}).
+update(SchemaMod, KeyPath, UpdateReq) ->
+    emqx_config_handler:update_config(SchemaMod, KeyPath, {update, UpdateReq}).

 -spec remove(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
-remove(KeyPath) ->
-    remove(emqx_schema, KeyPath).
+remove([RootName | _] = KeyPath) ->
+    remove(get_schema_mod(RootName), KeyPath).

-remove(SchemaModule, KeyPath) ->
-    emqx_config_handler:update_config(SchemaModule, KeyPath, remove).
+remove(SchemaMod, KeyPath) ->
+    emqx_config_handler:update_config(SchemaMod, KeyPath, remove).
+
+-spec reset(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
+reset([RootName | _] = KeyPath) ->
+    case get_default_value(KeyPath) of
+        {ok, Default} ->
+            emqx_config_handler:update_config(get_schema_mod(RootName), KeyPath,
+                {update, Default});
+        {error, _} = Error ->
+            Error
+    end.
+
+-spec get_default_value(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
+get_default_value([RootName | _] = KeyPath) ->
+    BinKeyPath = [bin(Key) || Key <- KeyPath],
+    case find_raw([RootName]) of
+        {ok, RawConf} ->
+            RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, RawConf),
+            SchemaMod = get_schema_mod(RootName),
+            try fill_defaults(SchemaMod, RawConf1) of FullConf ->
+                case emqx_map_lib:deep_find(BinKeyPath, FullConf) of
+                    {not_found, _, _} -> {error, no_default_value};
+                    {ok, Val} -> {ok, Val}
+                end
+            catch error:_ ->
+                {error, required_conf}
+            end;
+        {not_found, _, _} ->
+            {error, {rootname_not_found, RootName}}
+    end.

 -spec get_raw(emqx_map_lib:config_key_path()) -> term().
 get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath).
@ -208,7 +251,7 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config).
 %% NOTE: The order of the files is significant, configs from files orderd
 %% in the rear of the list overrides prior values.
 -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok.
-init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) ->
+init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
    ParseOptions = #{format => richmap},
    Parser = case is_binary(Conf) of
              true -> fun hocon:binary/2;
@ -216,39 +259,78 @@ init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) ->
             end,
    case Parser(Conf, ParseOptions) of
        {ok, RawRichConf} ->
-            init_load(SchemaModule, RawRichConf);
+            init_load(SchemaMod, RawRichConf);
        {error, Reason} ->
            logger:error(#{msg => failed_to_load_hocon_conf,
                           reason => Reason
                          }),
            error(failed_to_load_hocon_conf)
    end;
-init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) ->
+init_load(SchemaMod, RawRichConf) when is_map(RawRichConf) ->
    %% check with richmap for line numbers in error reports (future enhancement)
    Opts = #{return_plain => true,
             nullable => true
            },
    %% this call throws exception in case of check failure
-    {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, Opts),
-    ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(CheckedConf),
-            hocon_schema:richmap_to_map(RawRichConf)).
+    {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawRichConf, Opts),
+    ok = save_schema_mod(SchemaMod),
+    ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(normalize_conf(CheckedConf)),
+            normalize_conf(hocon_schema:richmap_to_map(RawRichConf))).
+
+normalize_conf(Conf) ->
+    maps:with(get_root_names(), Conf).

 -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf}
    when AppEnvs :: app_envs(), CheckedConf :: config().
-check_config(SchemaModule, RawConf) ->
+check_config(SchemaMod, RawConf) ->
    Opts = #{return_plain => true,
             nullable => true,
             format => map
            },
    {AppEnvs, CheckedConf} =
-        hocon_schema:map_translate(SchemaModule, RawConf, Opts),
+        hocon_schema:map_translate(SchemaMod, RawConf, Opts),
    Conf = maps:with(maps:keys(RawConf), CheckedConf),
    {AppEnvs, emqx_map_lib:unsafe_atom_key_map(Conf)}.

+-spec fill_defaults(raw_config()) -> map().
+fill_defaults(RawConf) ->
+    RootNames = get_root_names(),
+    maps:fold(fun(Key, Conf, Acc) ->
+            SubMap = #{Key => Conf},
+            WithDefaults = case lists:member(Key, RootNames) of
+                true -> fill_defaults(get_schema_mod(Key), SubMap);
+                false -> SubMap
+            end,
+            maps:merge(Acc, WithDefaults)
+        end, #{}, RawConf).
+
+-spec fill_defaults(module(), raw_config()) -> map().
+fill_defaults(SchemaMod, RawConf) ->
+    hocon_schema:check_plain(SchemaMod, RawConf,
+        #{nullable => true, no_conversion => true}, [str(K) || K <- maps:keys(RawConf)]).
+
 -spec read_override_conf() -> raw_config().
 read_override_conf() ->
    load_hocon_file(emqx_override_conf_name(), map).

+-spec save_schema_mod(module()) -> ok.
+save_schema_mod(SchemaMod) ->
+    OldMods = get_schema_mod(),
+    NewMods = maps:from_list([{bin(RootName), SchemaMod} || RootName <- SchemaMod:structs()]),
+    persistent_term:put(?PERSIS_MOD_ROOTNAMES, maps:merge(OldMods, NewMods)).
+
+-spec get_schema_mod() -> #{binary() => atom()}.
+get_schema_mod() ->
+    persistent_term:get(?PERSIS_MOD_ROOTNAMES, #{}).
+
+-spec get_schema_mod(atom() | binary()) -> [module()].
+get_schema_mod(RootName) ->
+    maps:get(bin(RootName), get_schema_mod()).
+
+-spec get_root_names() -> [binary()].
+get_root_names() ->
+    maps:keys(get_schema_mod()).
+
 -spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}.
 save_configs(_AppEnvs, Conf, RawConf, OverrideConf) ->
    %% We may need also support hot config update for the apps that use application envs.
@ -338,10 +420,20 @@ do_deep_put(?RAW_CONF, KeyPath, Map, Value) ->

 atom(Bin) when is_binary(Bin) ->
    binary_to_existing_atom(Bin, latin1);
+atom(Str) when is_list(Str) ->
+    list_to_existing_atom(Str);
 atom(Atom) when is_atom(Atom) ->
    Atom.

+str(Bin) when is_binary(Bin) ->
+    binary_to_list(Bin);
+str(Str) when is_list(Str) ->
+    Str;
+str(Atom) when is_atom(Atom) ->
+    atom_to_list(Atom).
+
 bin(Bin) when is_binary(Bin) -> Bin;
+bin(Str) when is_list(Str) -> list_to_binary(Str);
 bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).

 conf_key(?CONF, RootName) ->
--- a/apps/emqx/src/emqx_map_lib.erl
+++ b/apps/emqx/src/emqx_map_lib.erl
@ -23,6 +23,8 @@
        , deep_merge/2
        , safe_atom_key_map/1
        , unsafe_atom_key_map/1
+        , jsonable_map/1
+        , deep_convert/2
        ]).

 -export_type([config_key/0, config_key_path/0]).
@ -97,21 +99,42 @@ deep_merge(BaseMap, NewMap) ->
        end, #{}, BaseMap),
    maps:merge(MergedBase, maps:with(NewKeys, NewMap)).

+-spec deep_convert(map(), fun((K::any(), V::any()) -> {K1::any(), V1::any()})) -> map().
+deep_convert(Map, ConvFun) when is_map(Map) ->
+    maps:fold(fun(K, V, Acc) ->
+            {K1, V1} = ConvFun(K, deep_convert(V, ConvFun)),
+            Acc#{K1 => V1}
+        end, #{}, Map);
+deep_convert(ListV, ConvFun) when is_list(ListV) ->
+    [deep_convert(V, ConvFun) || V <- ListV];
+deep_convert(Val, _) -> Val.
+
+-spec unsafe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 unsafe_atom_key_map(Map) ->
    covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).

+-spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 safe_atom_key_map(Map) ->
    covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).

+-spec jsonable_map(map()) -> map().
+jsonable_map(Map) ->
+    deep_convert(Map, fun(K, V) ->
+            {jsonable_value(K), jsonable_value(V)}
+        end).
+
+jsonable_value([]) -> [];
+jsonable_value(Val) when is_list(Val) ->
+    case io_lib:printable_unicode_list(Val) of
+        true -> unicode:characters_to_binary(Val);
+        false -> Val
+    end;
+jsonable_value(Val) ->
+    Val.
+
 %%---------------------------------------------------------------------------
-covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) ->
-    maps:fold(
-        fun(K, V, Acc) when is_binary(K) ->
-              Acc#{Conv(K) => covert_keys_to_atom(V, Conv)};
-           (K, V, Acc) when is_atom(K) ->
-              %% richmap keys
-              Acc#{K => covert_keys_to_atom(V, Conv)}
-        end, #{}, BinKeyMap);
-covert_keys_to_atom(ListV, Conv) when is_list(ListV) ->
-    [covert_keys_to_atom(V, Conv) || V <- ListV];
-covert_keys_to_atom(Val, _) -> Val.
+covert_keys_to_atom(BinKeyMap, Conv) ->
+    deep_convert(BinKeyMap, fun
+            (K, V) when is_atom(K) -> {K, V};
+            (K, V) when is_binary(K) -> {Conv(K), V}
+        end).
--- a/apps/emqx_machine/src/emqx_machine_schema.erl
+++ b/apps/emqx_machine/src/emqx_machine_schema.erl
@ -54,6 +54,7 @@
        , emqx_dashboard_schema
        , emqx_gateway_schema
        , emqx_prometheus_schema
+        , emqx_rule_engine_schema
        , emqx_exhook_schema
        ]).

--- a/rebar.config
+++ b/rebar.config
@ -61,7 +61,7 @@
    , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
    , {getopt, "1.0.2"}
    , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}}
    , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}}
    , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}}
    ]}.