Merge pull request #9608 from zmstone/1123-pin-hocon-0.33.0

1123 pin hocon 0.33.0
2022-12-26 19:22:21 +08:00 · 2022-12-26 19:22:21 +08:00 · 81b226a801
parent 1e3b0c3777 f93c22045d
commit 81b226a801
16 changed files with 43 additions and 22 deletions
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@ -29,7 +29,7 @@
    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
    {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
    {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
    {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@ -446,7 +446,7 @@ fill_defaults(RawConf, Opts) ->

 -spec fill_defaults(module(), raw_config(), hocon_tconf:opts()) -> map().
 fill_defaults(SchemaMod, RawConf, Opts0) ->
-    Opts = maps:merge(#{required => false, only_fill_defaults => true}, Opts0),
+    Opts = maps:merge(#{required => false, make_serializable => true}, Opts0),
    hocon_tconf:check_plain(
        SchemaMod,
        RawConf,
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@ -59,7 +59,7 @@

 -export([
    validate_heap_size/1,
-    parse_user_lookup_fun/1,
+    user_lookup_fun_tr/2,
    validate_alarm_actions/1,
    non_empty_string/1,
    validations/0
@ -1931,7 +1931,7 @@ common_ssl_opts_schema(Defaults) ->
                typerefl:alias("string", any()),
                #{
                    default => <<"emqx_tls_psk:lookup">>,
-                    converter => fun ?MODULE:parse_user_lookup_fun/1,
+                    converter => fun ?MODULE:user_lookup_fun_tr/2,
                    desc => ?DESC(common_ssl_opts_schema_user_lookup_fun)
                }
            )},
@ -2277,6 +2277,19 @@ validate_alarm_actions(Actions) ->
        Error -> {error, Error}
    end.

+user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
+    fmt_user_lookup_fun(Lookup);
+user_lookup_fun_tr(Lookup, _) ->
+    parse_user_lookup_fun(Lookup).
+
+fmt_user_lookup_fun({Fun, _}) when is_function(Fun, 3) ->
+    {module, Mod} = erlang:fun_info(Fun, module),
+    {name, Name} = erlang:fun_info(Fun, name),
+    atom_to_list(Mod) ++ ":" ++ atom_to_list(Name);
+fmt_user_lookup_fun(Other) ->
+    %% already serializable
+    Other.
+
 parse_user_lookup_fun({Fun, _} = Lookup) when is_function(Fun, 3) -> Lookup;
 parse_user_lookup_fun(StrConf) ->
    [ModStr, FunStr] = string:tokens(str(StrConf), ": "),
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@ -1133,7 +1133,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) ->
 fill_defaults(Configs) when is_list(Configs) ->
    lists:map(fun fill_defaults/1, Configs);
 fill_defaults(Config) ->
-    emqx_authn:check_config(merge_default_headers(Config), #{only_fill_defaults => true}).
+    emqx_authn:check_config(merge_default_headers(Config), #{make_serializable => true}).

 merge_default_headers(Config) ->
    case maps:find(<<"headers">>, Config) of
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
    {description, "An OTP application"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
    {registered, []},
    {mod, {emqx_authz_app, []}},
    {applications, [
--- a/apps/emqx_authz/src/emqx_authz_api_sources.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl
@ -451,8 +451,7 @@ get_raw_sources() ->
    RawSources = emqx:get_raw_config([authorization, sources], []),
    Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
    Conf = #{<<"sources">> => RawSources},
-    Options = #{only_fill_defaults => true},
-    #{<<"sources">> := Sources} = hocon_tconf:check_plain(Schema, Conf, Options),
+    #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}),
    merge_default_headers(Sources).

 merge_default_headers(Sources) ->
--- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
@ -161,12 +161,13 @@ t_lookups(_Config) ->
        ]
    ).

-t_create_invalid(_Config) ->
+%% should still succeed to create even if the config will not work,
+%% because it's not a part of the schema check
+t_create_with_config_values_wont_works(_Config) ->
    AuthzConfig = raw_redis_authz_config(),

    InvalidConfigs =
        [
-            maps:without([<<"server">>], AuthzConfig),
            AuthzConfig#{<<"server">> => <<"unknownhost:3333">>},
            AuthzConfig#{<<"password">> => <<"wrongpass">>},
            AuthzConfig#{<<"database">> => <<"5678">>}
@ -180,6 +181,15 @@ t_create_invalid(_Config) ->
        InvalidConfigs
    ).

+%% creating without a require filed should return error
+t_create_invalid_schema(_Config) ->
+    AuthzConfig = raw_redis_authz_config(),
+    C = maps:without([<<"server">>], AuthzConfig),
+    ?assertMatch(
+        {error, {emqx_conf_schema, _}},
+        emqx_authz:update(?CMD_REPLACE, [C])
+    ).
+
 t_redis_error(_Config) ->
    ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}),

--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@ -26,7 +26,6 @@
 ]).

 -export([
-    ip_port_to_string/1,
    parse_server/2
 ]).

@ -105,6 +104,7 @@ password(type) -> binary();
 password(desc) -> ?DESC("password");
 password(required) -> false;
 password(format) -> <<"password">>;
+password(sensitive) -> true;
 password(_) -> undefined.

 auto_reconnect(type) -> boolean();
@ -112,11 +112,6 @@ auto_reconnect(desc) -> ?DESC("auto_reconnect");
 auto_reconnect(default) -> true;
 auto_reconnect(_) -> undefined.

-ip_port_to_string({Ip, Port}) when is_list(Ip) ->
-    iolist_to_binary([Ip, ":", integer_to_list(Port)]);
-ip_port_to_string({Ip, Port}) when is_tuple(Ip) ->
-    iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]).
-
 parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) ->
    case string:tokens(str(Str), ": ") of
        [Ip, Port] ->
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
    {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.7"},
+    {vsn, "5.0.8"},
    {modules, []},
    {registered, []},
    {mod, {emqx_exhook_app, []}},
--- a/apps/emqx_exhook/src/emqx_exhook_api.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_api.erl
@ -484,8 +484,7 @@ get_raw_config() ->
    RawConfig = emqx:get_raw_config([exhook, servers], []),
    Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
    Conf = #{<<"servers">> => RawConfig},
-    Options = #{only_fill_defaults => true},
-    #{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
+    #{<<"servers">> := Servers} = hocon_tconf:make_serializable(Schema, Conf, #{}),
    Servers.

 position_example() ->
--- a/apps/emqx_resource/src/emqx_resource_validator.erl
+++ b/apps/emqx_resource/src/emqx_resource_validator.erl
@ -30,6 +30,7 @@ min(Type, Min) ->

 not_empty(ErrMsg) ->
    fun
+        (undefined) -> {error, ErrMsg};
        (<<>>) -> {error, ErrMsg};
        (_) -> ok
    end.
--- a/changes/v5.0.13-en.md
+++ b/changes/v5.0.13-en.md
@ -24,3 +24,5 @@
  Prior to this change, a 'sticky' subscriber may continue to receive messages after unsubscribing.

 - Add check to ensure that a given key is among the prepared statements on query in the mysql connector [#9571](https://github.com/emqx/emqx/pull/9571).
+
+- Fix password leak to logs for connectors [#9608](https://github.com/emqx/emqx/pull/9608).
--- a/changes/v5.0.13-zh.md
+++ b/changes/v5.0.13-zh.md
@ -24,3 +24,5 @@
  在此修复前，使用 'sticky' 策略的订阅客户端可能在取消订阅后继续收到消息。

 - 增强 mysql 查询流程，确保给定的查询语句在 mysql 连接器的预编译语句中 [#9571](https://github.com/emqx/emqx/pull/9571)。
+
+- 修复连接器日志的密码泄漏 [#9608](https://github.com/emqx/emqx/pull/9608)。
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@ -1,5 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
       , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
       , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
--- a/mix.exs
+++ b/mix.exs
@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
      # in conflict by emqtt and hocon
      {:getopt, "1.0.2", override: true},
      {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
      {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
      {:esasl, github: "emqx/esasl", tag: "0.2.0"},
      {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
--- a/rebar.config
+++ b/rebar.config
@ -68,7 +68,7 @@
    , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
    , {getopt, "1.0.2"}
    , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
    , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
    , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
    , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}