From 6292d7ec683ff39b69d56220d96010cc620030e3 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 22 Dec 2022 14:48:02 +0100
Subject: [PATCH 1/7] refactor: remove dead code

---
 apps/emqx_connector/src/emqx_connector_schema_lib.erl | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index 53643c9f9..a8639fefa 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -26,7 +26,6 @@
 ]).
 
 -export([
-    ip_port_to_string/1,
     parse_server/2
 ]).
 
@@ -112,11 +111,6 @@ auto_reconnect(desc) -> ?DESC("auto_reconnect");
 auto_reconnect(default) -> true;
 auto_reconnect(_) -> undefined.
 
-ip_port_to_string({Ip, Port}) when is_list(Ip) ->
-    iolist_to_binary([Ip, ":", integer_to_list(Port)]);
-ip_port_to_string({Ip, Port}) when is_tuple(Ip) ->
-    iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]).
-
 parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) ->
     case string:tokens(str(Str), ": ") of
         [Ip, Port] ->

From 59970272ffbd4acedec5e83c543e28745244b50a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Dec 2022 11:57:07 +0100
Subject: [PATCH 2/7] chore: pin hocon 0.33.0

---
 apps/emqx/rebar.config             | 2 +-
 lib-ee/emqx_ee_bridge/rebar.config | 2 +-
 mix.exs                            | 2 +-
 rebar.config                       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 4c00d7f23..8ff4fea58 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index ee112def8..d551bb1b5 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,5 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
        , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
diff --git a/mix.exs b/mix.exs
index aea9c5d2e..1bf30486e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
diff --git a/rebar.config b/rebar.config
index 40614a443..861046ad9 100644
--- a/rebar.config
+++ b/rebar.config
@@ -68,7 +68,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

From fa7433628cb94c42ad48a54b2b9155cc6dd78339 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Thu, 22 Dec 2022 13:08:43 +0300
Subject: [PATCH 3/7] fix(logs): set password field sensitive for connectors

---
 apps/emqx_connector/src/emqx_connector_schema_lib.erl | 1 +
 changes/v5.0.13-en.md                                 | 2 ++
 changes/v5.0.13-zh.md                                 | 2 ++
 3 files changed, 5 insertions(+)

diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index a8639fefa..7bcfb6d21 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -104,6 +104,7 @@ password(type) -> binary();
 password(desc) -> ?DESC("password");
 password(required) -> false;
 password(format) -> <<"password">>;
+password(sensitive) -> true;
 password(_) -> undefined.
 
 auto_reconnect(type) -> boolean();
diff --git a/changes/v5.0.13-en.md b/changes/v5.0.13-en.md
index 38ca0a967..24f09d6f6 100644
--- a/changes/v5.0.13-en.md
+++ b/changes/v5.0.13-en.md
@@ -22,3 +22,5 @@
   Prior to this change, a 'sticky' subscriber may continue to receive messages after unsubscribing.
 
 - Add check to ensure that a given key is among the prepared statements on query in the mysql connector [#9571](https://github.com/emqx/emqx/pull/9571).
+
+- Fix password leak to logs for connectors [#9608](https://github.com/emqx/emqx/pull/9608).
diff --git a/changes/v5.0.13-zh.md b/changes/v5.0.13-zh.md
index 619803dfc..089324f8d 100644
--- a/changes/v5.0.13-zh.md
+++ b/changes/v5.0.13-zh.md
@@ -22,3 +22,5 @@
   在此修复前，使用 'sticky' 策略的订阅客户端可能在取消订阅后继续收到消息。
 
 - 增强 mysql 查询流程，确保给定的查询语句在 mysql 连接器的预编译语句中 [#9571](https://github.com/emqx/emqx/pull/9571)。
+
+- 修复连接器日志的密码泄漏 [#9608](https://github.com/emqx/emqx/pull/9608)。

From 350023e75798693e7d5b9a2729725634313955d0 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Dec 2022 13:58:31 +0100
Subject: [PATCH 4/7] fix(config): option only_fill_defaults renamed to
 make_serializable

---
 apps/emqx/src/emqx_config.erl                  | 2 +-
 apps/emqx_authn/src/emqx_authn_api.erl         | 2 +-
 apps/emqx_authz/src/emqx_authz_api_sources.erl | 3 +--
 apps/emqx_exhook/src/emqx_exhook_api.erl       | 3 +--
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index 16d9f31a7..f5ff64d4d 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -446,7 +446,7 @@ fill_defaults(RawConf, Opts) ->
 
 -spec fill_defaults(module(), raw_config(), hocon_tconf:opts()) -> map().
 fill_defaults(SchemaMod, RawConf, Opts0) ->
-    Opts = maps:merge(#{required => false, only_fill_defaults => true}, Opts0),
+    Opts = maps:merge(#{required => false, make_serializable => true}, Opts0),
     hocon_tconf:check_plain(
         SchemaMod,
         RawConf,
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index 8f67ba210..8898f1d4c 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -1133,7 +1133,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) ->
 fill_defaults(Configs) when is_list(Configs) ->
     lists:map(fun fill_defaults/1, Configs);
 fill_defaults(Config) ->
-    emqx_authn:check_config(merge_default_headers(Config), #{only_fill_defaults => true}).
+    emqx_authn:check_config(merge_default_headers(Config), #{make_serialiable => true}).
 
 merge_default_headers(Config) ->
     case maps:find(<<"headers">>, Config) of
diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl
index 3af2988db..3e67d4749 100644
--- a/apps/emqx_authz/src/emqx_authz_api_sources.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl
@@ -451,8 +451,7 @@ get_raw_sources() ->
     RawSources = emqx:get_raw_config([authorization, sources], []),
     Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
     Conf = #{<<"sources">> => RawSources},
-    Options = #{only_fill_defaults => true},
-    #{<<"sources">> := Sources} = hocon_tconf:check_plain(Schema, Conf, Options),
+    #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}),
     merge_default_headers(Sources).
 
 merge_default_headers(Sources) ->
diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl
index 6676c4503..c342ea66a 100644
--- a/apps/emqx_exhook/src/emqx_exhook_api.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_api.erl
@@ -484,8 +484,7 @@ get_raw_config() ->
     RawConfig = emqx:get_raw_config([exhook, servers], []),
     Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
     Conf = #{<<"servers">> => RawConfig},
-    Options = #{only_fill_defaults => true},
-    #{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
+    #{<<"servers">> := Servers} = hocon_tconf:make_serializable(Schema, Conf, #{}),
     Servers.
 
 position_example() ->

From d3efb0c0ba07e347aba3fd948ed10b48005afabb Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Dec 2022 15:10:16 +0100
Subject: [PATCH 5/7] chore: bump app versions

---
 apps/emqx_authz/src/emqx_authz.app.src   | 2 +-
 apps/emqx_exhook/src/emqx_exhook.app.src | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 3f0f96e72..c876fbf16 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
index 4e8d3d514..d08feb5e5 100644
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.7"},
+    {vsn, "5.0.8"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},

From 269a40a9ea222e379e8b75db8448152005881663 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Dec 2022 16:17:24 +0100
Subject: [PATCH 6/7] fix: format psk user lookup fun back to string

---
 apps/emqx/src/emqx_schema.erl          | 17 +++++++++++++++--
 apps/emqx_authn/src/emqx_authn_api.erl |  2 +-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index b1afcac7f..ebe27f2a5 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -59,7 +59,7 @@
 
 -export([
     validate_heap_size/1,
-    parse_user_lookup_fun/1,
+    user_lookup_fun_tr/2,
     validate_alarm_actions/1,
     non_empty_string/1,
     validations/0
@@ -1931,7 +1931,7 @@ common_ssl_opts_schema(Defaults) ->
                 typerefl:alias("string", any()),
                 #{
                     default => <<"emqx_tls_psk:lookup">>,
-                    converter => fun ?MODULE:parse_user_lookup_fun/1,
+                    converter => fun ?MODULE:user_lookup_fun_tr/2,
                     desc => ?DESC(common_ssl_opts_schema_user_lookup_fun)
                 }
             )},
@@ -2277,6 +2277,19 @@ validate_alarm_actions(Actions) ->
         Error -> {error, Error}
     end.
 
+user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
+    fmt_user_lookup_fun(Lookup);
+user_lookup_fun_tr(Lookup, _) ->
+    parse_user_lookup_fun(Lookup).
+
+fmt_user_lookup_fun({Fun, _}) when is_function(Fun, 3) ->
+    {module, Mod} = erlang:fun_info(Fun, module),
+    {name, Name} = erlang:fun_info(Fun, name),
+    atom_to_list(Mod) ++ ":" ++ atom_to_list(Name);
+fmt_user_lookup_fun(Other) ->
+    %% already serializable
+    Other.
+
 parse_user_lookup_fun({Fun, _} = Lookup) when is_function(Fun, 3) -> Lookup;
 parse_user_lookup_fun(StrConf) ->
     [ModStr, FunStr] = string:tokens(str(StrConf), ": "),
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index 8898f1d4c..2a5695cef 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -1133,7 +1133,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) ->
 fill_defaults(Configs) when is_list(Configs) ->
     lists:map(fun fill_defaults/1, Configs);
 fill_defaults(Config) ->
-    emqx_authn:check_config(merge_default_headers(Config), #{make_serialiable => true}).
+    emqx_authn:check_config(merge_default_headers(Config), #{make_serializable => true}).
 
 merge_default_headers(Config) ->
     case maps:find(<<"headers">>, Config) of

From f93c22045d2b61c5b968b06162f720cb2efa3ae5 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sat, 24 Dec 2022 11:39:27 +0100
Subject: [PATCH 7/7] fix: non-empty field should not be undefined

---
 apps/emqx_authz/test/emqx_authz_redis_SUITE.erl    | 14 ++++++++++++--
 apps/emqx_resource/src/emqx_resource_validator.erl |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
index ebf2b4d06..54d7a406c 100644
--- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
@@ -161,12 +161,13 @@ t_lookups(_Config) ->
         ]
     ).
 
-t_create_invalid(_Config) ->
+%% should still succeed to create even if the config will not work,
+%% because it's not a part of the schema check
+t_create_with_config_values_wont_works(_Config) ->
     AuthzConfig = raw_redis_authz_config(),
 
     InvalidConfigs =
         [
-            maps:without([<<"server">>], AuthzConfig),
             AuthzConfig#{<<"server">> => <<"unknownhost:3333">>},
             AuthzConfig#{<<"password">> => <<"wrongpass">>},
             AuthzConfig#{<<"database">> => <<"5678">>}
@@ -180,6 +181,15 @@ t_create_invalid(_Config) ->
         InvalidConfigs
     ).
 
+%% creating without a require filed should return error
+t_create_invalid_schema(_Config) ->
+    AuthzConfig = raw_redis_authz_config(),
+    C = maps:without([<<"server">>], AuthzConfig),
+    ?assertMatch(
+        {error, {emqx_conf_schema, _}},
+        emqx_authz:update(?CMD_REPLACE, [C])
+    ).
+
 t_redis_error(_Config) ->
     ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}),
 
diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl
index 7623ae7fa..a5404a65e 100644
--- a/apps/emqx_resource/src/emqx_resource_validator.erl
+++ b/apps/emqx_resource/src/emqx_resource_validator.erl
@@ -30,6 +30,7 @@ min(Type, Min) ->
 
 not_empty(ErrMsg) ->
     fun
+        (undefined) -> {error, ErrMsg};
         (<<>>) -> {error, ErrMsg};
         (_) -> ok
     end.