Merge pull request #9608 from zmstone/1123-pin-hocon-0.33.0

1123 pin hocon 0.33.0

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

apps/emqx/src/emqx_config.erl

@@ -446,7 +446,7 @@ fill_defaults(RawConf, Opts) ->
 
 -spec fill_defaults(module(), raw_config(), hocon_tconf:opts()) -> map().
 fill_defaults(SchemaMod, RawConf, Opts0) ->
-    Opts = maps:merge(#{required => false, only_fill_defaults => true}, Opts0),
+    Opts = maps:merge(#{required => false, make_serializable => true}, Opts0),
     hocon_tconf:check_plain(
         SchemaMod,
         RawConf,

apps/emqx/src/emqx_schema.erl

@@ -59,7 +59,7 @@
 
 -export([
     validate_heap_size/1,
-    parse_user_lookup_fun/1,
+    user_lookup_fun_tr/2,
     validate_alarm_actions/1,
     non_empty_string/1,
     validations/0
@@ -1931,7 +1931,7 @@ common_ssl_opts_schema(Defaults) ->
                 typerefl:alias("string", any()),
                 #{
                     default => <<"emqx_tls_psk:lookup">>,
-                    converter => fun ?MODULE:parse_user_lookup_fun/1,
+                    converter => fun ?MODULE:user_lookup_fun_tr/2,
                     desc => ?DESC(common_ssl_opts_schema_user_lookup_fun)
                 }
             )},
@@ -2277,6 +2277,19 @@ validate_alarm_actions(Actions) ->
         Error -> {error, Error}
     end.
 
+user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
+    fmt_user_lookup_fun(Lookup);
+user_lookup_fun_tr(Lookup, _) ->
+    parse_user_lookup_fun(Lookup).
+
+fmt_user_lookup_fun({Fun, _}) when is_function(Fun, 3) ->
+    {module, Mod} = erlang:fun_info(Fun, module),
+    {name, Name} = erlang:fun_info(Fun, name),
+    atom_to_list(Mod) ++ ":" ++ atom_to_list(Name);
+fmt_user_lookup_fun(Other) ->
+    %% already serializable
+    Other.
+
 parse_user_lookup_fun({Fun, _} = Lookup) when is_function(Fun, 3) -> Lookup;
 parse_user_lookup_fun(StrConf) ->
     [ModStr, FunStr] = string:tokens(str(StrConf), ": "),

apps/emqx_authn/src/emqx_authn_api.erl

@@ -1133,7 +1133,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) ->
 fill_defaults(Configs) when is_list(Configs) ->
     lists:map(fun fill_defaults/1, Configs);
 fill_defaults(Config) ->
-    emqx_authn:check_config(merge_default_headers(Config), #{only_fill_defaults => true}).
+    emqx_authn:check_config(merge_default_headers(Config), #{make_serializable => true}).
 
 merge_default_headers(Config) ->
     case maps:find(<<"headers">>, Config) of

apps/emqx_authz/src/emqx_authz.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [

apps/emqx_authz/src/emqx_authz_api_sources.erl

@@ -451,8 +451,7 @@ get_raw_sources() ->
     RawSources = emqx:get_raw_config([authorization, sources], []),
     Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
     Conf = #{<<"sources">> => RawSources},
-    Options = #{only_fill_defaults => true},
+    #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}),
-    #{<<"sources">> := Sources} = hocon_tconf:check_plain(Schema, Conf, Options),
     merge_default_headers(Sources).
 
 merge_default_headers(Sources) ->

apps/emqx_authz/test/emqx_authz_redis_SUITE.erl

@@ -161,12 +161,13 @@ t_lookups(_Config) ->
         ]
     ).
 
-t_create_invalid(_Config) ->
+%% should still succeed to create even if the config will not work,
+%% because it's not a part of the schema check
+t_create_with_config_values_wont_works(_Config) ->
     AuthzConfig = raw_redis_authz_config(),
 
     InvalidConfigs =
         [
-            maps:without([<<"server">>], AuthzConfig),
             AuthzConfig#{<<"server">> => <<"unknownhost:3333">>},
             AuthzConfig#{<<"password">> => <<"wrongpass">>},
             AuthzConfig#{<<"database">> => <<"5678">>}
@@ -180,6 +181,15 @@ t_create_invalid(_Config) ->
         InvalidConfigs
     ).
 
+%% creating without a require filed should return error
+t_create_invalid_schema(_Config) ->
+    AuthzConfig = raw_redis_authz_config(),
+    C = maps:without([<<"server">>], AuthzConfig),
+    ?assertMatch(
+        {error, {emqx_conf_schema, _}},
+        emqx_authz:update(?CMD_REPLACE, [C])
+    ).
+
 t_redis_error(_Config) ->
     ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}),
 

apps/emqx_connector/src/emqx_connector_schema_lib.erl

@@ -26,7 +26,6 @@
 ]).
 
 -export([
-    ip_port_to_string/1,
     parse_server/2
 ]).
 
@@ -105,6 +104,7 @@ password(type) -> binary();
 password(desc) -> ?DESC("password");
 password(required) -> false;
 password(format) -> <<"password">>;
+password(sensitive) -> true;
 password(_) -> undefined.
 
 auto_reconnect(type) -> boolean();
@@ -112,11 +112,6 @@ auto_reconnect(desc) -> ?DESC("auto_reconnect");
 auto_reconnect(default) -> true;
 auto_reconnect(_) -> undefined.
 
-ip_port_to_string({Ip, Port}) when is_list(Ip) ->
-    iolist_to_binary([Ip, ":", integer_to_list(Port)]);
-ip_port_to_string({Ip, Port}) when is_tuple(Ip) ->
-    iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]).
-
 parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) ->
     case string:tokens(str(Str), ": ") of
         [Ip, Port] ->

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.7"},
+    {vsn, "5.0.8"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook_api.erl

@@ -484,8 +484,7 @@ get_raw_config() ->
     RawConfig = emqx:get_raw_config([exhook, servers], []),
     Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
     Conf = #{<<"servers">> => RawConfig},
-    Options = #{only_fill_defaults => true},
+    #{<<"servers">> := Servers} = hocon_tconf:make_serializable(Schema, Conf, #{}),
-    #{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
     Servers.
 
 position_example() ->

apps/emqx_resource/src/emqx_resource_validator.erl

@@ -30,6 +30,7 @@ min(Type, Min) ->
 
 not_empty(ErrMsg) ->
     fun
+        (undefined) -> {error, ErrMsg};
         (<<>>) -> {error, ErrMsg};
         (_) -> ok
     end.

changes/v5.0.13-en.md

@@ -24,3 +24,5 @@
   Prior to this change, a 'sticky' subscriber may continue to receive messages after unsubscribing.
 
 - Add check to ensure that a given key is among the prepared statements on query in the mysql connector [#9571](https://github.com/emqx/emqx/pull/9571).
+
+- Fix password leak to logs for connectors [#9608](https://github.com/emqx/emqx/pull/9608).

changes/v5.0.13-zh.md

@@ -24,3 +24,5 @@
   在此修复前，使用 'sticky' 策略的订阅客户端可能在取消订阅后继续收到消息。
 
 - 增强 mysql 查询流程，确保给定的查询语句在 mysql 连接器的预编译语句中 [#9571](https://github.com/emqx/emqx/pull/9571)。
+
+- 修复连接器日志的密码泄漏 [#9608](https://github.com/emqx/emqx/pull/9608)。

lib-ee/emqx_ee_bridge/rebar.config

@@ -1,5 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
        , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}

mix.exs

@@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -68,7 +68,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}