From 2d7099e3ae559f8acead6069a6c81ac2be2fd751 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 13 Dec 2022 09:26:47 +0100 Subject: [PATCH 1/4] refactor(emqx_bridge_resource): rename a variable --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ad35485ed..ef5dd6093 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -298,8 +298,8 @@ parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) -> %% For some drivers that can be used as data-sources, we need to provide a %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it %% receives a message from the external database. - BName = bridge_id(Type, Name), - Conf#{hookpoint => <<"$bridges/", BName/binary>>, bridge_name => Name}; + BId = bridge_id(Type, Name), + Conf#{hookpoint => <<"$bridges/", BId/binary>>, bridge_name => Name}; parse_confs(_Type, _Name, Conf) -> Conf. From 34f7fff97d4a37b174084d0bd2fd5373b7a20493 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 13 Dec 2022 09:26:05 +0100 Subject: [PATCH 2/4] chore: pin hocon 0.32.0 --- apps/emqx/rebar.config | 2 +- lib-ee/emqx_ee_bridge/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d91b32da0..ff27566ec 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 9119b052d..ee112def8 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,5 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}} +{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} diff --git a/mix.exs b/mix.exs index 7aa632bd9..11ef14eb1 100644 --- a/mix.exs +++ b/mix.exs @@ -67,7 +67,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.31.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 3e1eafc0f..e9678b1cd 100644 --- a/rebar.config +++ b/rebar.config @@ -67,7 +67,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From abef881a733794eaed11215838826a40d01578d9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Dec 2022 11:52:54 +0100 Subject: [PATCH 3/4] refactor(config): no need to explicitly remove env metadata after upgraded to honcon 0.32.0, the env meta is removed by default. --- apps/emqx/src/emqx_config.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index fa1c63868..1f9a46c03 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -575,10 +575,10 @@ load_hocon_file(FileName, LoadType) -> end. do_get_raw(Path) -> - hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path)). + do_get(?RAW_CONF, Path). do_get_raw(Path, Default) -> - hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path, Default)). + do_get(?RAW_CONF, Path, Default). do_get(Type, KeyPath) -> Ref = make_ref(), From 7137a422dfc76d09b5fed4d34f53eeb8522ea583 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Dec 2022 11:04:10 +0100 Subject: [PATCH 4/4] refactor(emqx_authn): make error messages more readable --- apps/emqx/src/emqx_config.erl | 1 + apps/emqx_authn/src/emqx_authn.erl | 44 ++++++++++++++++-------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 1f9a46c03..16d9f31a7 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -402,6 +402,7 @@ merge_envs(SchemaMod, RawConf) -> required => false, format => map, apply_override_envs => true, + remove_env_meta => true, check_lazy => true }, hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts). diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index ac0ee0eb0..c7f9d6fdf 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -39,12 +39,15 @@ providers() -> {{scram, built_in_database}, emqx_enhanced_authn_scram_mnesia} ]. -check_configs(C) when is_map(C) -> - check_configs([C]); -check_configs([]) -> +check_configs(CM) when is_map(CM) -> + check_configs([CM]); +check_configs(CL) -> + check_configs(CL, 1). + +check_configs([], _Nth) -> []; -check_configs([Config | Configs]) -> - [check_config(Config) | check_configs(Configs)]. +check_configs([Config | Configs], Nth) -> + [check_config(Config, #{id_for_log => Nth}) | check_configs(Configs, Nth + 1)]. check_config(Config) -> check_config(Config, #{}). @@ -55,15 +58,16 @@ check_config(Config, Opts) -> #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. -do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> +do_check_config(#{<<"mechanism">> := Mec0} = Config, Opts) -> + Mec = atom(Mec0, #{error => unknown_mechanism}), Key = case maps:get(<<"backend">>, Config, false) of - false -> atom(Mec); - Backend -> {atom(Mec), atom(Backend)} + false -> Mec; + Backend -> {Mec, atom(Backend, #{error => unknown_backend})} end, case lists:keyfind(Key, 1, providers()) of false -> - throw({unknown_handler, Key}); + throw(#{error => unknown_authn_provider, which => Key}); {_, ProviderModule} -> hocon_tconf:check_plain( ProviderModule, @@ -71,22 +75,22 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> Opts#{atom_key => true} ) end; -do_check_config(Config, _Opts) when is_map(Config) -> - throw({invalid_config, "mechanism_field_required", Config}); -do_check_config(RawConf, Opts) -> - %% authentication conf is lazy type, when it comes from ENV, it is a string - %% EMQX_AUTHENTICATION__1="{mechanism=\"password_based\"...}" - case hocon:binary(RawConf, Opts) of - {ok, Conf} -> do_check_config(Conf, Opts); - {error, Reason} -> throw({invalid_config, Reason}) - end. +do_check_config(Config, Opts) when is_map(Config) -> + throw(#{ + error => invalid_config, + which => maps:get(id_for_log, Opts, unknown), + reason => "mechanism_field_required" + }). -atom(Bin) -> +%% The atoms have to be loaded already, +%% which might be an issue for plugins which are loaded after node boot +%% but they should really manage their own configs in that case. +atom(Bin, ErrorContext) -> try binary_to_existing_atom(Bin, utf8) catch _:_ -> - throw({unknown_auth_provider, Bin}) + throw(ErrorContext#{value => Bin}) end. -spec get_enabled_authns() ->