From 1ffb7f5f9443e34ce88872e22e8b3a013dd91885 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 28 Apr 2022 16:18:10 +0800 Subject: [PATCH 1/3] fix: remove '\$FROM_ENV_VAR' from raw conf --- apps/emqx/src/emqx_config.erl | 4 +-- apps/emqx/src/emqx_hocon.erl | 65 ++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index a59d6a670..7bef29827 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -261,10 +261,10 @@ get_default_value([RootName | _] = KeyPath) -> end. -spec get_raw(emqx_map_lib:config_key_path()) -> term(). -get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath). +get_raw(KeyPath) -> emqx_hocon:remove_env_meta(do_get(?RAW_CONF, KeyPath)). -spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). -get_raw(KeyPath, Default) -> do_get(?RAW_CONF, KeyPath, Default). +get_raw(KeyPath, Default) -> emqx_hocon:remove_env_meta(do_get(?RAW_CONF, KeyPath, Default)). -spec put_raw(map()) -> ok. put_raw(Config) -> diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 77e040721..850d5a033 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -19,9 +19,13 @@ -export([ format_path/1, - check/2 + check/2, + remove_env_meta/1 ]). +%% FIXME: move this to hoconsc.hrl +-define(FROM_ENV_VAR(Name, Value), {'$FROM_ENV_VAR', Name, Value}). + %% @doc Format hocon config field path to dot-separated string in iolist format. -spec format_path([atom() | string() | binary()]) -> iolist(). format_path([]) -> ""; @@ -51,7 +55,66 @@ check(SchemaModule, HoconText) -> {error, Reason} end. +%% @doc remove FROM_ENV_VAR from value +remove_env_meta(Map) when is_map(Map) -> + remove_env_meta(maps:iterator(Map), #{}); +remove_env_meta(Array) when is_list(Array) -> + [remove_env_meta(R) || R <- Array]; +remove_env_meta(Value) -> + Value. + +remove_env_meta(Iter, Map) -> + case maps:next(Iter) of + {K, ?FROM_ENV_VAR(_Env, Val), I} -> + remove_env_meta(I, Map#{K => Val}); + {K, V, I} when is_binary(V) -> + remove_env_meta(I, Map#{K => V}); + {K, V, I} -> + remove_env_meta(I, Map#{K => remove_env_meta(V)}); + none -> + Map + end. + %% Ensure iolist() iol(B) when is_binary(B) -> B; iol(A) when is_atom(A) -> atom_to_binary(A, utf8); iol(L) when is_list(L) -> L. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +make_keys_test() -> + Seq = [ + { + #{<<"k1">> => <<"v1">>}, + #{<<"k1">> => <<"v1">>} + }, + { + #{<<"k1">> => ?FROM_ENV_VAR("V1", <<"v1">>)}, + #{<<"k1">> => <<"v1">>} + }, + { + #{<<"k1">> => #{<<"k2">> => <<"v1">>}}, + #{<<"k1">> => #{<<"k2">> => <<"v1">>}} + }, + { + #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", <<"v1">>)}}, + #{<<"k1">> => #{<<"k2">> => <<"v1">>}} + }, + { + #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", <<"v1">>), <<"k3">> => <<"v3">>}}, + #{<<"k1">> => #{<<"k2">> => <<"v1">>, <<"k3">> => <<"v3">>}} + }, + { + #{<<"k1">> => #{<<"k2">> => 1024}}, + #{<<"k1">> => #{<<"k2">> => 1024}} + }, + { + #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", 1024)}}, + #{<<"k1">> => #{<<"k2">> => 1024}} + } + ], + lists:foreach(fun({Data, Expect}) -> ?assertEqual(Expect, remove_env_meta(Data)) end, Seq), + ok. + +-endif. From 6f1610257fe747f9ca28d46098af8fc6eb58d4c8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 29 Apr 2022 08:48:12 +0800 Subject: [PATCH 2/3] feat: bump hocon to 0.27.5 to remove FROM_ENV --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 4 +- apps/emqx/src/emqx_hocon.erl | 65 +------------------------------ apps/emqx_prometheus/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 7 insertions(+), 70 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 458326b81..4342c159c 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.1"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.4"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.4"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 7bef29827..22b9eb05e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -261,10 +261,10 @@ get_default_value([RootName | _] = KeyPath) -> end. -spec get_raw(emqx_map_lib:config_key_path()) -> term(). -get_raw(KeyPath) -> emqx_hocon:remove_env_meta(do_get(?RAW_CONF, KeyPath)). +get_raw(KeyPath) -> hocon_tconf:remove_env_meta(do_get(?RAW_CONF, KeyPath)). -spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). -get_raw(KeyPath, Default) -> emqx_hocon:remove_env_meta(do_get(?RAW_CONF, KeyPath, Default)). +get_raw(KeyPath, Default) -> hocon_tconf:remove_env_meta(do_get(?RAW_CONF, KeyPath, Default)). -spec put_raw(map()) -> ok. put_raw(Config) -> diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 850d5a033..77e040721 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -19,13 +19,9 @@ -export([ format_path/1, - check/2, - remove_env_meta/1 + check/2 ]). -%% FIXME: move this to hoconsc.hrl --define(FROM_ENV_VAR(Name, Value), {'$FROM_ENV_VAR', Name, Value}). - %% @doc Format hocon config field path to dot-separated string in iolist format. -spec format_path([atom() | string() | binary()]) -> iolist(). format_path([]) -> ""; @@ -55,66 +51,7 @@ check(SchemaModule, HoconText) -> {error, Reason} end. -%% @doc remove FROM_ENV_VAR from value -remove_env_meta(Map) when is_map(Map) -> - remove_env_meta(maps:iterator(Map), #{}); -remove_env_meta(Array) when is_list(Array) -> - [remove_env_meta(R) || R <- Array]; -remove_env_meta(Value) -> - Value. - -remove_env_meta(Iter, Map) -> - case maps:next(Iter) of - {K, ?FROM_ENV_VAR(_Env, Val), I} -> - remove_env_meta(I, Map#{K => Val}); - {K, V, I} when is_binary(V) -> - remove_env_meta(I, Map#{K => V}); - {K, V, I} -> - remove_env_meta(I, Map#{K => remove_env_meta(V)}); - none -> - Map - end. - %% Ensure iolist() iol(B) when is_binary(B) -> B; iol(A) when is_atom(A) -> atom_to_binary(A, utf8); iol(L) when is_list(L) -> L. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -make_keys_test() -> - Seq = [ - { - #{<<"k1">> => <<"v1">>}, - #{<<"k1">> => <<"v1">>} - }, - { - #{<<"k1">> => ?FROM_ENV_VAR("V1", <<"v1">>)}, - #{<<"k1">> => <<"v1">>} - }, - { - #{<<"k1">> => #{<<"k2">> => <<"v1">>}}, - #{<<"k1">> => #{<<"k2">> => <<"v1">>}} - }, - { - #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", <<"v1">>)}}, - #{<<"k1">> => #{<<"k2">> => <<"v1">>}} - }, - { - #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", <<"v1">>), <<"k3">> => <<"v3">>}}, - #{<<"k1">> => #{<<"k2">> => <<"v1">>, <<"k3">> => <<"v3">>}} - }, - { - #{<<"k1">> => #{<<"k2">> => 1024}}, - #{<<"k1">> => #{<<"k2">> => 1024}} - }, - { - #{<<"k1">> => #{<<"k2">> => ?FROM_ENV_VAR("V1", 1024)}}, - #{<<"k1">> => #{<<"k2">> => 1024}} - } - ], - lists:foreach(fun({Data, Expect}) -> ?assertEqual(Expect, remove_env_meta(Data)) end, Seq), - ok. - --endif. diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 910ee9f82..67e664b0a 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -4,7 +4,7 @@ {emqx, {path, "../emqx"}}, %% FIXME: tag this as v3.1.3 {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.4"}}} + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/mix.exs b/mix.exs index b87579a10..5ad69b7d6 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.27.4", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.27.5", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 54114c608..4c60b5e68 100644 --- a/rebar.config +++ b/rebar.config @@ -66,7 +66,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.4"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From e24bdab2a55ea3b300a3e5f5121876a624a3241c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 29 Apr 2022 08:52:03 +0800 Subject: [PATCH 3/3] fix: crash when swagger hocon translation failed --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e84d6f0af..9924b085c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -820,11 +820,14 @@ serialize_hocon_error_msg({_Schema, Errors}) -> serialize_hocon_error_msg(Error) -> iolist_to_binary(io_lib:format("~p", [Error])). -hocon_error({validation_error, #{path := Path} = Error}) -> +hocon_error({Type, #{path := Path} = Error}) -> Error1 = maps:without([path, stacktrace], Error), - emqx_logger_jsonfmt:best_effort_json(Error1#{<<"path">> => sub_path(Path)}, []); -hocon_error({translation_error, #{value_path := Path} = Error}) -> - Error1 = maps:without([value_path, stacktrace], Error), - emqx_logger_jsonfmt:best_effort_json(Error1#{<<"path">> => sub_path(Path)}, []). + emqx_logger_jsonfmt:best_effort_json( + Error1#{ + <<"path">> => sub_path(Path), + <<"type">> => Type + }, + [] + ). sub_path(Path) -> string:trim(Path, leading, "root.").