diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 4c00d7f23..8ff4fea58 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 16d9f31a7..f5ff64d4d 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -446,7 +446,7 @@ fill_defaults(RawConf, Opts) -> -spec fill_defaults(module(), raw_config(), hocon_tconf:opts()) -> map(). fill_defaults(SchemaMod, RawConf, Opts0) -> - Opts = maps:merge(#{required => false, only_fill_defaults => true}, Opts0), + Opts = maps:merge(#{required => false, make_serializable => true}, Opts0), hocon_tconf:check_plain( SchemaMod, RawConf, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index b1afcac7f..ebe27f2a5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -59,7 +59,7 @@ -export([ validate_heap_size/1, - parse_user_lookup_fun/1, + user_lookup_fun_tr/2, validate_alarm_actions/1, non_empty_string/1, validations/0 @@ -1931,7 +1931,7 @@ common_ssl_opts_schema(Defaults) -> typerefl:alias("string", any()), #{ default => <<"emqx_tls_psk:lookup">>, - converter => fun ?MODULE:parse_user_lookup_fun/1, + converter => fun ?MODULE:user_lookup_fun_tr/2, desc => ?DESC(common_ssl_opts_schema_user_lookup_fun) } )}, @@ -2277,6 +2277,19 @@ validate_alarm_actions(Actions) -> Error -> {error, Error} end. +user_lookup_fun_tr(Lookup, #{make_serializable := true}) -> + fmt_user_lookup_fun(Lookup); +user_lookup_fun_tr(Lookup, _) -> + parse_user_lookup_fun(Lookup). + +fmt_user_lookup_fun({Fun, _}) when is_function(Fun, 3) -> + {module, Mod} = erlang:fun_info(Fun, module), + {name, Name} = erlang:fun_info(Fun, name), + atom_to_list(Mod) ++ ":" ++ atom_to_list(Name); +fmt_user_lookup_fun(Other) -> + %% already serializable + Other. + parse_user_lookup_fun({Fun, _} = Lookup) when is_function(Fun, 3) -> Lookup; parse_user_lookup_fun(StrConf) -> [ModStr, FunStr] = string:tokens(str(StrConf), ": "), diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 8f67ba210..2a5695cef 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1133,7 +1133,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) -> fill_defaults(Configs) when is_list(Configs) -> lists:map(fun fill_defaults/1, Configs); fill_defaults(Config) -> - emqx_authn:check_config(merge_default_headers(Config), #{only_fill_defaults => true}). + emqx_authn:check_config(merge_default_headers(Config), #{make_serializable => true}). merge_default_headers(Config) -> case maps:find(<<"headers">>, Config) of diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 3f0f96e72..c876fbf16 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 3af2988db..3e67d4749 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -451,8 +451,7 @@ get_raw_sources() -> RawSources = emqx:get_raw_config([authorization, sources], []), Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, Conf = #{<<"sources">> => RawSources}, - Options = #{only_fill_defaults => true}, - #{<<"sources">> := Sources} = hocon_tconf:check_plain(Schema, Conf, Options), + #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}), merge_default_headers(Sources). merge_default_headers(Sources) -> diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index ebf2b4d06..54d7a406c 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -161,12 +161,13 @@ t_lookups(_Config) -> ] ). -t_create_invalid(_Config) -> +%% should still succeed to create even if the config will not work, +%% because it's not a part of the schema check +t_create_with_config_values_wont_works(_Config) -> AuthzConfig = raw_redis_authz_config(), InvalidConfigs = [ - maps:without([<<"server">>], AuthzConfig), AuthzConfig#{<<"server">> => <<"unknownhost:3333">>}, AuthzConfig#{<<"password">> => <<"wrongpass">>}, AuthzConfig#{<<"database">> => <<"5678">>} @@ -180,6 +181,15 @@ t_create_invalid(_Config) -> InvalidConfigs ). +%% creating without a require filed should return error +t_create_invalid_schema(_Config) -> + AuthzConfig = raw_redis_authz_config(), + C = maps:without([<<"server">>], AuthzConfig), + ?assertMatch( + {error, {emqx_conf_schema, _}}, + emqx_authz:update(?CMD_REPLACE, [C]) + ). + t_redis_error(_Config) -> ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}), diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 53643c9f9..7bcfb6d21 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -26,7 +26,6 @@ ]). -export([ - ip_port_to_string/1, parse_server/2 ]). @@ -105,6 +104,7 @@ password(type) -> binary(); password(desc) -> ?DESC("password"); password(required) -> false; password(format) -> <<"password">>; +password(sensitive) -> true; password(_) -> undefined. auto_reconnect(type) -> boolean(); @@ -112,11 +112,6 @@ auto_reconnect(desc) -> ?DESC("auto_reconnect"); auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. -ip_port_to_string({Ip, Port}) when is_list(Ip) -> - iolist_to_binary([Ip, ":", integer_to_list(Port)]); -ip_port_to_string({Ip, Port}) when is_tuple(Ip) -> - iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]). - parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) -> case string:tokens(str(Str), ": ") of [Ip, Port] -> diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 4e8d3d514..d08feb5e5 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index 6676c4503..c342ea66a 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -484,8 +484,7 @@ get_raw_config() -> RawConfig = emqx:get_raw_config([exhook, servers], []), Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}}, Conf = #{<<"servers">> => RawConfig}, - Options = #{only_fill_defaults => true}, - #{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options), + #{<<"servers">> := Servers} = hocon_tconf:make_serializable(Schema, Conf, #{}), Servers. position_example() -> diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index 7623ae7fa..a5404a65e 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -30,6 +30,7 @@ min(Type, Min) -> not_empty(ErrMsg) -> fun + (undefined) -> {error, ErrMsg}; (<<>>) -> {error, ErrMsg}; (_) -> ok end. diff --git a/changes/v5.0.13-en.md b/changes/v5.0.13-en.md index 3381f3a4a..06329c574 100644 --- a/changes/v5.0.13-en.md +++ b/changes/v5.0.13-en.md @@ -24,3 +24,5 @@ Prior to this change, a 'sticky' subscriber may continue to receive messages after unsubscribing. - Add check to ensure that a given key is among the prepared statements on query in the mysql connector [#9571](https://github.com/emqx/emqx/pull/9571). + +- Fix password leak to logs for connectors [#9608](https://github.com/emqx/emqx/pull/9608). diff --git a/changes/v5.0.13-zh.md b/changes/v5.0.13-zh.md index 4a5b0f67b..386ea4746 100644 --- a/changes/v5.0.13-zh.md +++ b/changes/v5.0.13-zh.md @@ -24,3 +24,5 @@ 在此修复前,使用 'sticky' 策略的订阅客户端可能在取消订阅后继续收到消息。 - 增强 mysql 查询流程,确保给定的查询语句在 mysql 连接器的预编译语句中 [#9571](https://github.com/emqx/emqx/pull/9571)。 + +- 修复连接器日志的密码泄漏 [#9608](https://github.com/emqx/emqx/pull/9608)。 diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index ee112def8..d551bb1b5 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,5 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} +{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} diff --git a/mix.exs b/mix.exs index aea9c5d2e..1bf30486e 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 40614a443..861046ad9 100644 --- a/rebar.config +++ b/rebar.config @@ -68,7 +68,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}