From 0eed01abee30a7c934b940cd6f89e0642328e1f9 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 22 Sep 2023 16:03:28 +0800
Subject: [PATCH 1/6] fix: create ssl listener return 500 crash

---
 .../src/emqx_mgmt_api_listeners.erl           | 36 ++++++++++---------
 .../test/emqx_mgmt_api_listeners_SUITE.erl    | 14 ++++++--
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
index 90fb1f98e..8295047b9 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
@@ -266,7 +266,7 @@ fields(node_status) ->
             })},
         {status, ?HOCON(?R_REF(status))}
     ];
-fields({Type, with_name}) ->
+fields("with_name_" ++ Type) ->
     listener_struct_with_name(Type);
 fields(Type) ->
     listener_struct(Type).
@@ -308,7 +308,7 @@ listener_union_member_selector(Opts) ->
 
 create_listener_schema(Opts) ->
     Schemas = [
-        ?R_REF(Mod, {Type, with_name})
+        ?R_REF(Mod, "with_name_" ++ Type)
      || #{ref := ?R_REF(Mod, Type)} <- listeners_info(Opts)
     ],
     Example = maps:remove(id, tcp_schema_example()),
@@ -399,7 +399,7 @@ list_listeners(get, #{query_string := Query}) ->
         end,
     {200, listener_status_by_id(NodeL)};
 list_listeners(post, #{body := Body}) ->
-    create_listener(Body).
+    create_listener(name, Body).
 
 crud_listeners_by_id(get, #{bindings := #{id := Id}}) ->
     case find_listeners_by_id(Id) of
@@ -407,7 +407,7 @@ crud_listeners_by_id(get, #{bindings := #{id := Id}}) ->
         [L] -> {200, L}
     end;
 crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) ->
-    case parse_listener_conf(Body0) of
+    case parse_listener_conf(id, Body0) of
         {Id, Type, Name, Conf} ->
             case get_raw(Type, Name) of
                 undefined ->
@@ -430,7 +430,7 @@ crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) ->
             {400, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_ID_INCONSISTENT}}
     end;
 crud_listeners_by_id(post, #{body := Body}) ->
-    create_listener(Body);
+    create_listener(id, Body);
 crud_listeners_by_id(delete, #{bindings := #{id := Id}}) ->
     {ok, #{type := Type, name := Name}} = emqx_listeners:parse_listener_id(Id),
     case find_listeners_by_id(Id) of
@@ -441,11 +441,10 @@ crud_listeners_by_id(delete, #{bindings := #{id := Id}}) ->
             {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}
     end.
 
-parse_listener_conf(Conf0) ->
+parse_listener_conf(id, Conf0) ->
     Conf1 = maps:without([<<"running">>, <<"current_connections">>], Conf0),
     {TypeBin, Conf2} = maps:take(<<"type">>, Conf1),
     TypeAtom = binary_to_existing_atom(TypeBin),
-
     case maps:take(<<"id">>, Conf2) of
         {IdBin, Conf3} ->
             {ok, #{type := Type, name := Name}} = emqx_listeners:parse_listener_id(IdBin),
@@ -454,13 +453,18 @@ parse_listener_conf(Conf0) ->
                 false -> {error, listener_type_inconsistent}
             end;
         _ ->
-            case maps:take(<<"name">>, Conf2) of
-                {Name, Conf3} ->
-                    IdBin = <<TypeBin/binary, $:, Name/binary>>,
-                    {binary_to_atom(IdBin), TypeAtom, Name, Conf3};
-                _ ->
-                    {error, listener_config_invalid}
-            end
+            {error, listener_config_invalid}
+    end;
+parse_listener_conf(name, Conf0) ->
+    Conf1 = maps:without([<<"running">>, <<"current_connections">>], Conf0),
+    {TypeBin, Conf2} = maps:take(<<"type">>, Conf1),
+    TypeAtom = binary_to_existing_atom(TypeBin),
+    case maps:take(<<"name">>, Conf2) of
+        {Name, Conf3} ->
+            IdBin = <<TypeBin/binary, $:, Name/binary>>,
+            {binary_to_atom(IdBin), TypeAtom, Name, Conf3};
+        _ ->
+            {error, listener_config_invalid}
     end.
 
 stop_listeners_by_id(Method, Body = #{bindings := Bindings}) ->
@@ -832,8 +836,8 @@ tcp_schema_example() ->
         type => tcp
     }.
 
-create_listener(Body) ->
-    case parse_listener_conf(Body) of
+create_listener(From, Body) ->
+    case parse_listener_conf(From, Body) of
         {Id, Type, Name, Conf} ->
             case create(Type, Name, Conf) of
                 {ok, #{raw_config := _RawConf}} ->
diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
index 96ec8f2de..53579406f 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
@@ -238,7 +238,6 @@ t_clear_certs(Config) when is_list(Config) ->
     NewConf2 = emqx_utils_maps:deep_put(
         [<<"ssl_options">>, <<"keyfile">>], NewConf, cert_file("keyfile")
     ),
-
     _ = request(post, NewPath, [], NewConf2),
     ListResult1 = list_pem_dir("ssl", "clear"),
     ?assertMatch({ok, [_, _]}, ListResult1),
@@ -251,7 +250,7 @@ t_clear_certs(Config) when is_list(Config) ->
     _ = emqx_tls_certfile_gc:force(),
     ListResult2 = list_pem_dir("ssl", "clear"),
 
-    %% make sure the old cret file is deleted
+    %% make sure the old cert file is deleted
     ?assertMatch({ok, [_, _]}, ListResult2),
 
     {ok, ResultList1} = ListResult1,
@@ -273,6 +272,17 @@ t_clear_certs(Config) when is_list(Config) ->
     _ = delete(NewPath),
     _ = emqx_tls_certfile_gc:force(),
     ?assertMatch({error, enoent}, list_pem_dir("ssl", "clear")),
+
+    %% test create listeners without id in path
+    NewPath1 = emqx_mgmt_api_test_util:api_path(["listeners"]),
+    NewConf3 = maps:remove(<<"id">>, NewConf2#{<<"name">> => <<"clear">>}),
+    ?assertNotMatch({error, {"HTTP/1.1", 400, _}}, request(post, NewPath1, [], NewConf3)),
+    ListResult3 = list_pem_dir("ssl", "clear"),
+    ?assertMatch({ok, [_, _]}, ListResult3),
+    _ = delete(NewPath),
+    _ = emqx_tls_certfile_gc:force(),
+    ?assertMatch({error, enoent}, list_pem_dir("ssl", "clear")),
+
     ok.
 
 get_tcp_listeners(Node) ->

From ff7f37ccf5e38fafdd6d77a8652308b3594faeeb Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 25 Sep 2023 13:22:41 -0300
Subject: [PATCH 2/6] test(cth): allow defining schema to load for app

---
 apps/emqx/test/emqx_cth_suite.erl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl
index 24105b2b4..5cbca3243 100644
--- a/apps/emqx/test/emqx_cth_suite.erl
+++ b/apps/emqx/test/emqx_cth_suite.erl
@@ -52,7 +52,7 @@
 %%    (e.g. in `init_per_suite/1` / `init_per_group/2`), providing the appspecs
 %%    and unique work dir for the testrun (e.g. `work_dir/1`). Save the result
 %%    in a context.
-%% 3. Call `emqx_cth_sutie:stop/1` to stop the applications after the testrun
+%% 3. Call `emqx_cth_suite:stop/1` to stop the applications after the testrun
 %%    finishes (e.g. in `end_per_suite/1` / `end_per_group/2`), providing the
 %%    result from step 2.
 -module(emqx_cth_suite).
@@ -245,6 +245,9 @@ spec_fmt(ffun, {_, X}) -> X.
 
 maybe_configure_app(_App, #{config := false}) ->
     ok;
+maybe_configure_app(_App, AppConfig = #{schema_mod := SchemaModule}) when is_atom(SchemaModule) ->
+    #{config := Config} = AppConfig,
+    configure_app(SchemaModule, Config);
 maybe_configure_app(App, #{config := Config}) ->
     case app_schema(App) of
         {ok, SchemaModule} ->

From 5d212e1086e9dcbc31001bd5c4e6be039866977f Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 25 Sep 2023 13:23:39 -0300
Subject: [PATCH 3/6] fix(audit): only support audit log on enterprise edition

Fixes https://emqx.atlassian.net/browse/EMQX-11039
---
 apps/emqx_conf/src/emqx_conf_schema.erl       | 57 +-------------
 .../emqx_conf/test/emqx_conf_logger_SUITE.erl | 11 +--
 .../emqx_conf/test/emqx_conf_schema_tests.erl | 19 +----
 .../src/emqx_enterprise_schema.erl            | 77 ++++++++++++++++++-
 .../test/emqx_enterprise_schema_SUITE.erl     | 52 +++++++++++++
 .../test/emqx_enterprise_schema_tests.erl     | 35 +++++++++
 rel/i18n/emqx_conf_schema.hocon               |  3 -
 7 files changed, 169 insertions(+), 85 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index f1bfc3d31..27799fa93 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -43,6 +43,9 @@
 ]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 
+%% internal exports for `emqx_enterprise_schema' only.
+-export([ensure_unicode_path/2, convert_rotation/2, log_handler_common_confs/2]).
+
 %% Static apps which merge their configs into the merged emqx.conf
 %% The list can not be made a dynamic read at run-time as it is used
 %% by nodetool to generate app.<time>.config before EMQX is started
@@ -962,15 +965,6 @@ fields("log") ->
                     aliases => [file_handlers],
                     importance => ?IMPORTANCE_HIGH
                 }
-            )},
-        {"audit",
-            sc(
-                ?R_REF("log_audit_handler"),
-                #{
-                    desc => ?DESC("log_audit_handler"),
-                    importance => ?IMPORTANCE_HIGH,
-                    default => #{<<"enable">> => true, <<"level">> => <<"info">>}
-                }
             )}
     ];
 fields("console_handler") ->
@@ -1012,49 +1006,6 @@ fields("log_file_handler") ->
                 }
             )}
     ] ++ log_handler_common_confs(file, #{});
-fields("log_audit_handler") ->
-    [
-        {"path",
-            sc(
-                file(),
-                #{
-                    desc => ?DESC("audit_file_handler_path"),
-                    default => <<"${EMQX_LOG_DIR}/audit.log">>,
-                    importance => ?IMPORTANCE_HIGH,
-                    converter => fun(Path, Opts) ->
-                        emqx_schema:naive_env_interpolation(ensure_unicode_path(Path, Opts))
-                    end
-                }
-            )},
-        {"rotation_count",
-            sc(
-                range(1, 128),
-                #{
-                    default => 10,
-                    converter => fun convert_rotation/2,
-                    desc => ?DESC("log_rotation_count"),
-                    importance => ?IMPORTANCE_MEDIUM
-                }
-            )},
-        {"rotation_size",
-            sc(
-                hoconsc:union([infinity, emqx_schema:bytesize()]),
-                #{
-                    default => <<"50MB">>,
-                    desc => ?DESC("log_file_handler_max_size"),
-                    importance => ?IMPORTANCE_MEDIUM
-                }
-            )}
-    ] ++
-        %% Only support json
-        lists:keydelete(
-            "formatter",
-            1,
-            log_handler_common_confs(
-                file,
-                #{level => info, level_desc => "audit_handler_level"}
-            )
-        );
 fields("log_overload_kill") ->
     [
         {"enable",
@@ -1145,8 +1096,6 @@ desc("console_handler") ->
     ?DESC("desc_console_handler");
 desc("log_file_handler") ->
     ?DESC("desc_log_file_handler");
-desc("log_audit_handler") ->
-    ?DESC("desc_audit_log_handler");
 desc("log_rotation") ->
     ?DESC("desc_log_rotation");
 desc("log_overload_kill") ->
diff --git a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl
index c31c155d0..a53a81565 100644
--- a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl
+++ b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl
@@ -78,16 +78,7 @@ t_log_conf(_Conf) ->
                 <<"time_offset">> => <<"system">>
             },
         <<"file">> =>
-            #{<<"default">> => FileExpect},
-        <<"audit">> =>
-            #{
-                <<"enable">> => true,
-                <<"level">> => <<"info">>,
-                <<"path">> => <<"log/audit.log">>,
-                <<"rotation_count">> => 10,
-                <<"rotation_size">> => <<"50MB">>,
-                <<"time_offset">> => <<"system">>
-            }
+            #{<<"default">> => FileExpect}
     },
     ?assertEqual(ExpectLog1, emqx_conf:get_raw([<<"log">>])),
     UpdateLog0 = emqx_utils_maps:deep_remove([<<"file">>, <<"default">>], ExpectLog1),
diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl
index b0e6a304c..1faaa2109 100644
--- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl
+++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl
@@ -181,23 +181,8 @@ validate_log(Conf) ->
         }},
         FileHandler
     ),
-    AuditHandler = lists:keyfind(emqx_audit, 2, FileHandlers),
-    %% default is enable and log level is info.
-    ?assertMatch(
-        {handler, emqx_audit, logger_disk_log_h, #{
-            config := #{
-                type := wrap,
-                file := "log/audit.log",
-                max_no_bytes := _,
-                max_no_files := _
-            },
-            filesync_repeat_interval := no_repeat,
-            filters := [{filter_audit, {_, stop}}],
-            formatter := _,
-            level := info
-        }},
-        AuditHandler
-    ),
+    %% audit is an EE-only feature
+    ?assertNot(lists:keyfind(emqx_audit, 2, FileHandlers)),
     ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers),
     ?assertEqual(
         {handler, console, logger_std_h, #{
diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl
index a801825e0..9c381d928 100644
--- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl
+++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl
@@ -6,6 +6,9 @@
 
 -behaviour(hocon_schema).
 
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
 -export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]).
 
 -define(EE_SCHEMA_MODULES, [
@@ -22,6 +25,53 @@ roots() ->
 
 fields("node") ->
     redefine_node(emqx_conf_schema:fields("node"));
+fields("log") ->
+    redefine_log(emqx_conf_schema:fields("log"));
+fields("log_audit_handler") ->
+    [
+        {"path",
+            hoconsc:mk(
+                emqx_conf_schema:file(),
+                #{
+                    desc => ?DESC(emqx_conf_schema, "audit_file_handler_path"),
+                    default => <<"${EMQX_LOG_DIR}/audit.log">>,
+                    importance => ?IMPORTANCE_HIGH,
+                    converter => fun(Path, Opts) ->
+                        emqx_schema:naive_env_interpolation(
+                            emqx_conf_schema:ensure_unicode_path(Path, Opts)
+                        )
+                    end
+                }
+            )},
+        {"rotation_count",
+            hoconsc:mk(
+                range(1, 128),
+                #{
+                    default => 10,
+                    converter => fun emqx_conf_schema:convert_rotation/2,
+                    desc => ?DESC(emqx_conf_schema, "log_rotation_count"),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )},
+        {"rotation_size",
+            hoconsc:mk(
+                hoconsc:union([infinity, emqx_schema:bytesize()]),
+                #{
+                    default => <<"50MB">>,
+                    desc => ?DESC(emqx_conf_schema, "log_file_handler_max_size"),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )}
+    ] ++
+        %% Only support json
+        lists:keydelete(
+            "formatter",
+            1,
+            emqx_conf_schema:log_handler_common_confs(
+                file,
+                #{level => info, level_desc => "audit_handler_level"}
+            )
+        );
 fields(Name) ->
     ee_delegate(fields, ?EE_SCHEMA_MODULES, Name).
 
@@ -31,6 +81,8 @@ translations() ->
 translation(Name) ->
     emqx_conf_schema:translation(Name).
 
+desc("log_audit_handler") ->
+    ?DESC(emqx_conf_schema, "desc_audit_log_handler");
 desc(Name) ->
     ee_delegate(desc, ?EE_SCHEMA_MODULES, Name).
 
@@ -60,13 +112,20 @@ ee_delegate(Method, [], Name) ->
     apply(emqx_conf_schema, Method, [Name]).
 
 redefine_roots(Roots) ->
-    Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}],
+    Overrides = [
+        {"node", #{type => hoconsc:ref(?MODULE, "node")}},
+        {"log", #{type => hoconsc:ref(?MODULE, "log")}}
+    ],
     override(Roots, Overrides).
 
 redefine_node(Fields) ->
     Overrides = [],
     override(Fields, Overrides).
 
+redefine_log(Fields) ->
+    Overrides = [],
+    override(Fields, Overrides) ++ audit_log_conf().
+
 override(Fields, []) ->
     Fields;
 override(Fields, [{Name, Override} | More]) ->
@@ -81,3 +140,19 @@ find_schema(Name, Fields) ->
 
 replace_schema(Name, Schema, Fields) ->
     lists:keyreplace(Name, 1, Fields, {Name, Schema}).
+
+audit_log_conf() ->
+    [
+        {"audit",
+            hoconsc:mk(
+                hoconsc:ref(?MODULE, "log_audit_handler"),
+                #{
+                    %% note: we need to keep the descriptions associated with
+                    %% `emqx_conf_schema' module hocon i18n file because that's what
+                    %% `emqx_conf:gen_config_md' seems to expect.
+                    desc => ?DESC(emqx_conf_schema, "log_audit_handler"),
+                    importance => ?IMPORTANCE_HIGH,
+                    default => #{<<"enable">> => true, <<"level">> => <<"info">>}
+                }
+            )}
+    ].
diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl
index e7f4c0075..b0da7ba6b 100644
--- a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl
+++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl
@@ -13,6 +13,25 @@
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
+init_per_testcase(t_audit_log_conf, Config) ->
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_enterprise,
+            {emqx_conf, #{schema_mod => emqx_enterprise_schema}}
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config];
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_audit_log_conf, Config) ->
+    Apps = ?config(apps, Config),
+    ok = emqx_cth_suite:stop(Apps),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Tests
 %%------------------------------------------------------------------------------
@@ -50,3 +69,36 @@ t_translations(_Config) ->
         emqx_conf_schema:translation(Root),
         emqx_enterprise_schema:translation(Root)
     ).
+
+t_audit_log_conf(_Config) ->
+    FileExpect = #{
+        <<"enable">> => true,
+        <<"formatter">> => <<"text">>,
+        <<"level">> => <<"warning">>,
+        <<"rotation_count">> => 10,
+        <<"rotation_size">> => <<"50MB">>,
+        <<"time_offset">> => <<"system">>,
+        <<"path">> => <<"log/emqx.log">>
+    },
+    ExpectLog1 = #{
+        <<"console">> =>
+            #{
+                <<"enable">> => false,
+                <<"formatter">> => <<"text">>,
+                <<"level">> => <<"warning">>,
+                <<"time_offset">> => <<"system">>
+            },
+        <<"file">> =>
+            #{<<"default">> => FileExpect},
+        <<"audit">> =>
+            #{
+                <<"enable">> => true,
+                <<"level">> => <<"info">>,
+                <<"path">> => <<"log/audit.log">>,
+                <<"rotation_count">> => 10,
+                <<"rotation_size">> => <<"50MB">>,
+                <<"time_offset">> => <<"system">>
+            }
+    },
+    ?assertEqual(ExpectLog1, emqx_conf:get_raw([<<"log">>])),
+    ok.
diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl
index 955f358f3..15e3e4004 100644
--- a/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl
+++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl
@@ -16,3 +16,38 @@ doc_gen_test() ->
             ok = emqx_conf:dump_schema(Dir, emqx_enterprise_schema)
         end
     }.
+
+audit_log_test() ->
+    ensure_acl_conf(),
+    Conf0 = <<"node {cookie = aaa, data_dir = \"/tmp\"}">>,
+    {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}),
+    ConfList = hocon_tconf:generate(emqx_enterprise_schema, ConfMap0),
+    Kernel = proplists:get_value(kernel, ConfList),
+    Loggers = proplists:get_value(logger, Kernel),
+    FileHandlers = lists:filter(fun(L) -> element(3, L) =:= logger_disk_log_h end, Loggers),
+    AuditHandler = lists:keyfind(emqx_audit, 2, FileHandlers),
+    %% default is enable and log level is info.
+    ?assertMatch(
+        {handler, emqx_audit, logger_disk_log_h, #{
+            config := #{
+                type := wrap,
+                file := "log/audit.log",
+                max_no_bytes := _,
+                max_no_files := _
+            },
+            filesync_repeat_interval := no_repeat,
+            filters := [{filter_audit, {_, stop}}],
+            formatter := _,
+            level := info
+        }},
+        AuditHandler
+    ),
+    ok.
+
+ensure_acl_conf() ->
+    File = emqx_schema:naive_env_interpolation(<<"${EMQX_ETC_DIR}/acl.conf">>),
+    ok = filelib:ensure_dir(filename:dirname(File)),
+    case filelib:is_regular(File) of
+        true -> ok;
+        false -> file:write_file(File, <<"">>)
+    end.
diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon
index a6c158768..a026c6121 100644
--- a/rel/i18n/emqx_conf_schema.hocon
+++ b/rel/i18n/emqx_conf_schema.hocon
@@ -841,7 +841,4 @@ Defaults to 100000."""
 node_channel_cleanup_batch_size.label:
 """Node Channel Cleanup Batch Size"""
 
-prevent_overlapping_partitions.desc:
-"""https://www.erlang.org/doc/man/global.html#description"""
-
 }

From 8135e2758664555f54b6cb5f2e20745a936cea35 Mon Sep 17 00:00:00 2001
From: firest <enlanubo@gmail.com>
Date: Tue, 26 Sep 2023 11:06:47 +0800
Subject: [PATCH 4/6] fix: redact sensitive data in SSO and LDAP

---
 .../src/emqx_dashboard_sso_api.erl            | 22 ++++++++++++++-----
 apps/emqx_ldap/src/emqx_ldap.erl              |  9 +++++---
 apps/emqx_ldap/src/emqx_ldap_authz.erl        |  2 +-
 apps/emqx_ldap/src/emqx_ldap_bind_worker.erl  |  8 ++++---
 4 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl
index 5ac24f1f4..c9c9a7fd9 100644
--- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl
+++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl
@@ -159,17 +159,23 @@ login(post, #{bindings := #{backend := Backend}, body := Body} = Request) ->
         State ->
             case emqx_dashboard_sso:login(provider(Backend), Request, State) of
                 {ok, Role, Token} ->
-                    ?SLOG(info, #{msg => "dashboard_sso_login_successful", request => Request}),
+                    ?SLOG(info, #{
+                        msg => "dashboard_sso_login_successful",
+                        request => emqx_utils:redact(Request)
+                    }),
                     Username = maps:get(<<"username">>, Body),
                     {200, login_meta(Username, Role, Token)};
                 {redirect, Redirect} ->
-                    ?SLOG(info, #{msg => "dashboard_sso_login_redirect", request => Request}),
+                    ?SLOG(info, #{
+                        msg => "dashboard_sso_login_redirect",
+                        request => emqx_utils:redact(Request)
+                    }),
                     Redirect;
                 {error, Reason} ->
                     ?SLOG(info, #{
                         msg => "dashboard_sso_login_failed",
-                        request => Request,
-                        reason => Reason
+                        request => emqx_utils:redact(Request),
+                        reason => emqx_utils:redact(Reason)
                     }),
                     {401, #{code => ?BAD_USERNAME_OR_PWD, message => <<"Auth failed">>}}
             end
@@ -193,10 +199,14 @@ backend(get, #{bindings := #{backend := Type}}) ->
             {200, to_json(Backend)}
     end;
 backend(put, #{bindings := #{backend := Backend}, body := Config}) ->
-    ?SLOG(info, #{msg => "Update SSO backend", backend => Backend, config => Config}),
+    ?SLOG(info, #{
+        msg => "update_sso_backend",
+        backend => Backend,
+        config => emqx_utils:redact(Config)
+    }),
     on_backend_update(Backend, Config, fun emqx_dashboard_sso_manager:update/2);
 backend(delete, #{bindings := #{backend := Backend}}) ->
-    ?SLOG(info, #{msg => "Delete SSO backend", backend => Backend}),
+    ?SLOG(info, #{msg => "delete_sso_backend", backend => Backend}),
     handle_backend_update_result(emqx_dashboard_sso_manager:delete(Backend), undefined).
 
 sso_parameters(Params) ->
diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl
index f16166793..7b856521c 100644
--- a/apps/emqx_ldap/src/emqx_ldap.erl
+++ b/apps/emqx_ldap/src/emqx_ldap.erl
@@ -158,7 +158,7 @@ on_start(
         {error, Reason} ->
             ?tp(
                 ldap_connector_start_failed,
-                #{error => Reason}
+                #{error => emqx_utils:redact(Reason)}
             ),
             {error, Reason}
     end.
@@ -248,7 +248,7 @@ do_ldap_query(
     SearchOptions,
     #{pool_name := PoolName} = State
 ) ->
-    LogMeta = #{connector => InstId, search => SearchOptions, state => State},
+    LogMeta = #{connector => InstId, search => SearchOptions, state => emqx_utils:redact(State)},
     ?TRACE("QUERY", "ldap_connector_received", LogMeta),
     case
         ecpool:pick_and_do(
@@ -268,7 +268,10 @@ do_ldap_query(
         {error, Reason} ->
             ?SLOG(
                 error,
-                LogMeta#{msg => "ldap_connector_do_query_failed", reason => Reason}
+                LogMeta#{
+                    msg => "ldap_connector_do_query_failed",
+                    reason => emqx_utils:redact(Reason)
+                }
             ),
             {error, {unrecoverable_error, Reason}}
     end.
diff --git a/apps/emqx_ldap/src/emqx_ldap_authz.erl b/apps/emqx_ldap/src/emqx_ldap_authz.erl
index f5ef38c01..13110306c 100644
--- a/apps/emqx_ldap/src/emqx_ldap_authz.erl
+++ b/apps/emqx_ldap/src/emqx_ldap_authz.erl
@@ -116,7 +116,7 @@ authorize(
         {error, Reason} ->
             ?SLOG(error, #{
                 msg => "query_ldap_error",
-                reason => Reason,
+                reason => emqx_utils:redact(Reason),
                 resource_id => ResourceID
             }),
             nomatch
diff --git a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl
index e4e341002..5fb69f0fd 100644
--- a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl
+++ b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl
@@ -61,7 +61,7 @@ on_query(
     {bind, Data},
     #{
         base_tokens := DNTks,
-        bind_password_tokens := PWTks,
+        bind_password := PWTks,
         bind_pool_name := PoolName
     } = State
 ) ->
@@ -86,7 +86,7 @@ on_query(
         {error, Reason} ->
             ?SLOG(
                 error,
-                LogMeta#{msg => "ldap_bind_failed", reason => Reason}
+                LogMeta#{msg => "ldap_bind_failed", reason => emqx_utils:redact(Reason)}
             ),
             {error, {unrecoverable_error, Reason}}
     end.
@@ -100,7 +100,9 @@ prepare_template(Config, State) ->
     do_prepare_template(maps:to_list(maps:with([bind_password], Config)), State).
 
 do_prepare_template([{bind_password, V} | T], State) ->
-    do_prepare_template(T, State#{bind_password_tokens => emqx_placeholder:preproc_tmpl(V)});
+    %% This is sensitive data
+    %% to reduce match cases, here we reuse the existing sensitive filter key: bind_password
+    do_prepare_template(T, State#{bind_password => emqx_placeholder:preproc_tmpl(V)});
 do_prepare_template([], State) ->
     State.
 

From d1af81c86e9046020adbf301482a3031ba0cf26f Mon Sep 17 00:00:00 2001
From: firest <enlanubo@gmail.com>
Date: Tue, 26 Sep 2023 12:12:04 +0800
Subject: [PATCH 5/6] fix(sso): fix the failure to update SSO users

---
 apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 2 +-
 apps/emqx_dashboard/src/emqx_dashboard_api.erl   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
index e77204541..e262cf389 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
@@ -230,7 +230,7 @@ remove_user(Username) ->
 
 -spec update_user(dashboard_username(), dashboard_user_role(), binary()) ->
     {ok, map()} | {error, term()}.
-update_user(Username, Role, Desc) when is_binary(Username) ->
+update_user(Username, Role, Desc) ->
     case legal_role(Role) of
         ok ->
             case
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
index 70d6accb4..9ee07dc06 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
@@ -379,9 +379,9 @@ sso_parameters() ->
 sso_parameters(Params) ->
     emqx_dashboard_sso_api:sso_parameters(Params).
 
-username(#{bindings := #{backend := local}}, Username) ->
+username(#{query_string := #{<<"backend">> := local}}, Username) ->
     Username;
-username(#{bindings := #{backend := Backend}}, Username) ->
+username(#{query_string := #{<<"backend">> := Backend}}, Username) ->
     ?SSO_USERNAME(Backend, Username);
 username(_Req, Username) ->
     Username.

From 56917fee5fa94eb1ef8340021fbb9b9db3c5769a Mon Sep 17 00:00:00 2001
From: firest <enlanubo@gmail.com>
Date: Tue, 26 Sep 2023 14:15:41 +0800
Subject: [PATCH 6/6] fix(sso): Improved SSO user deletion command

---
 .../emqx_dashboard/include/emqx_dashboard.hrl |  1 +
 .../src/emqx_dashboard_admin.erl              |  2 +-
 .../emqx_dashboard/src/emqx_dashboard_api.erl |  2 +-
 .../src/emqx_dashboard_token.erl              |  2 +-
 .../src/emqx_dashboard_sso_cli.erl            | 30 +++++++++++--------
 5 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl
index 3e089ccbf..9013436e7 100644
--- a/apps/emqx_dashboard/include/emqx_dashboard.hrl
+++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl
@@ -24,6 +24,7 @@
 -define(ROLE_SUPERUSER, <<"administrator">>).
 -define(ROLE_DEFAULT, ?ROLE_SUPERUSER).
 
+-define(BACKEND_LOCAL, local).
 -define(SSO_USERNAME(Backend, Name), {Backend, Name}).
 
 -type dashboard_sso_backend() :: atom().
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
index e262cf389..e9aac164b 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
@@ -427,7 +427,7 @@ flatten_username(#{username := ?SSO_USERNAME(Backend, Name)} = Data) ->
         backend => Backend
     };
 flatten_username(#{username := Username} = Data) when is_binary(Username) ->
-    Data#{backend => local}.
+    Data#{backend => ?BACKEND_LOCAL}.
 
 -spec add_sso_user(dashboard_sso_backend(), binary(), dashboard_user_role(), binary()) ->
     {ok, map()} | {error, any()}.
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
index 9ee07dc06..8a81f2116 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
@@ -379,7 +379,7 @@ sso_parameters() ->
 sso_parameters(Params) ->
     emqx_dashboard_sso_api:sso_parameters(Params).
 
-username(#{query_string := #{<<"backend">> := local}}, Username) ->
+username(#{query_string := #{<<"backend">> := ?BACKEND_LOCAL}}, Username) ->
     Username;
 username(#{query_string := #{<<"backend">> := Backend}}, Username) ->
     ?SSO_USERNAME(Backend, Username);
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl
index 866da971b..1c840e90c 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl
@@ -191,7 +191,7 @@ token_ttl() ->
 format(Token, ?SSO_USERNAME(Backend, Name), Role, ExpTime) ->
     format(Token, Backend, Name, Role, ExpTime);
 format(Token, Username, Role, ExpTime) ->
-    format(Token, local, Username, Role, ExpTime).
+    format(Token, ?BACKEND_LOCAL, Username, Role, ExpTime).
 
 format(Token, Backend, Username, Role, ExpTime) ->
     #?ADMIN_JWT{
diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_cli.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_cli.erl
index 308d37f5c..2fe4f8937 100644
--- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_cli.erl
+++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_cli.erl
@@ -34,16 +34,13 @@ admins(["passwd", Username, Password]) ->
             print_error(Reason)
     end;
 admins(["del", Username]) ->
-    case emqx_dashboard_admin:remove_user(bin(Username)) of
-        {ok, _} ->
-            emqx_ctl:print("ok~n");
-        {error, Reason} ->
-            print_error(Reason)
-    end;
-admins(["del", Username, Backend]) ->
-    case emqx_dashboard_admin:remove_user(?SSO_USERNAME(atom(Backend), bin(Username))) of
-        {ok, _} ->
-            emqx_ctl:print("ok~n");
+    delete_user(bin(Username));
+admins(["del", Username, BackendName]) ->
+    case atom(BackendName) of
+        {ok, ?BACKEND_LOCAL} ->
+            delete_user(bin(Username));
+        {ok, Backend} ->
+            delete_user(?SSO_USERNAME(Backend, bin(Username)));
         {error, Reason} ->
             print_error(Reason)
     end;
@@ -52,9 +49,18 @@ admins(_) ->
         [
             {"admins add <Username> <Password> <Description> <Role>", "Add dashboard user"},
             {"admins passwd <Username> <Password>", "Reset dashboard user password"},
-            {"admins del <Username> <Backend>", "Delete dashboard user"}
+            {"admins del <Username> <Backend>",
+                "Delete dashboard user, <Backend> can be omitted, the default value is 'local'"}
         ]
     ).
 
 atom(S) ->
-    erlang:list_to_atom(S).
+    emqx_utils:safe_to_existing_atom(S).
+
+delete_user(Username) ->
+    case emqx_dashboard_admin:remove_user(Username) of
+        {ok, _} ->
+            emqx_ctl:print("ok~n");
+        {error, Reason} ->
+            print_error(Reason)
+    end.