From 9cbad5297c8ed03fb3eb4ec306d1718788d94095 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Wed, 1 Feb 2023 11:16:30 +0800
Subject: [PATCH 1/2] fix: primary logger level not update

---
 apps/emqx/src/config/emqx_config_logger.erl | 10 +++++++---
 apps/emqx/src/emqx_app.erl                  |  1 -
 apps/emqx_conf/src/emqx_conf_app.erl        |  1 +
 deploy/packages/emqx.service                |  2 +-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl
index 4d787ea0c..625a498e2 100644
--- a/apps/emqx/src/config/emqx_config_logger.erl
+++ b/apps/emqx/src/config/emqx_config_logger.erl
@@ -67,12 +67,16 @@ post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
     ok.
 
 maybe_update_log_level(NewLevel) ->
-    OldLevel = application:get_env(kernel, logger_level, warning),
+    OldLevel = emqx_logger:get_primary_log_level(),
     case OldLevel =:= NewLevel of
         true ->
             %% no change
             ok;
         false ->
+            ok = emqx_logger:set_primary_log_level(NewLevel),
+            %% also update kernel's logger_level for troubleshooting
+            %% what is actually in effect is the logger's primary log level
+            ok = application:set_env(kernel, logger_level, NewLevel),
             log_to_console("Config override: log level is set to '~p'~n", [NewLevel])
     end.
 
@@ -97,7 +101,7 @@ update_log_handlers(NewHandlers) ->
     end,
     AddsAndUpdates = lists:filtermap(MapFn, NewHandlers),
     lists:foreach(fun update_log_handler/1, Removes ++ AddsAndUpdates),
-    _ = application:set_env(kernel, logger, NewHandlers),
+    ok = application:set_env(kernel, logger, NewHandlers),
     ok.
 
 update_log_handler({removed, Id}) ->
@@ -115,7 +119,7 @@ id_for_log(Other) -> "log.file_handlers." ++ atom_to_list(Other).
 atom(Id) when is_binary(Id) -> binary_to_atom(Id, utf8);
 atom(Id) when is_atom(Id) -> Id.
 
-%% @doc Translate raw config to app-env conpatible log handler configs list.
+%% @doc Translate raw config to app-env compatible log handler configs list.
 tr_handlers(Conf) ->
     %% mute the default handler
     tr_console_handler(Conf) ++
diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl
index c56cdb520..6188d8030 100644
--- a/apps/emqx/src/emqx_app.erl
+++ b/apps/emqx/src/emqx_app.erl
@@ -42,7 +42,6 @@
 start(_Type, _Args) ->
     ok = maybe_load_config(),
     ok = emqx_persistent_session:init_db_backend(),
-    ok = emqx_config_logger:refresh_config(),
     ok = maybe_start_quicer(),
     ok = emqx_bpapi:start(),
     wait_boot_shards(),
diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl
index f2e4f6f56..34224c3f2 100644
--- a/apps/emqx_conf/src/emqx_conf_app.erl
+++ b/apps/emqx_conf/src/emqx_conf_app.erl
@@ -29,6 +29,7 @@
 
 start(_StartType, _StartArgs) ->
     init_conf(),
+    ok = emqx_config_logger:refresh_config(),
     emqx_conf_sup:start_link().
 
 stop(_State) ->
diff --git a/deploy/packages/emqx.service b/deploy/packages/emqx.service
index d505cc519..d826e358b 100644
--- a/deploy/packages/emqx.service
+++ b/deploy/packages/emqx.service
@@ -13,7 +13,7 @@ Environment=HOME=/var/lib/emqx
 # Enable logging to file
 Environment=EMQX_LOG__TO=default
 
-# Start 'foregroun' but not 'start' (daemon) mode.
+# Start 'foreground' but not 'start' (daemon) mode.
 # Because systemd monitor/restarts 'simple' services
 ExecStart=/bin/bash /usr/bin/emqx foreground
 

From 95ef1e806c293d2b376dbf69e709e9a64f48c9f9 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Wed, 1 Feb 2023 11:24:27 +0800
Subject: [PATCH 2/2] chore: don't crash when add_handler failed

---
 apps/emqx/src/config/emqx_config_logger.erl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl
index 625a498e2..15e4d3959 100644
--- a/apps/emqx/src/config/emqx_config_logger.erl
+++ b/apps/emqx/src/config/emqx_config_logger.erl
@@ -111,7 +111,17 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) ->
     log_to_console("Config override: ~s is ~p~n", [id_for_log(Id), Action]),
     % may return {error, {not_found, Id}}
     _ = logger:remove_handler(Id),
-    ok = logger:add_handler(Id, Mod, Conf).
+    case logger:add_handler(Id, Mod, Conf) of
+        ok ->
+            ok;
+        %% Don't crash here, otherwise the cluster rpc will retry the wrong handler forever.
+        {error, Reason} ->
+            log_to_console(
+                "Config override: ~s is ~p, but failed to add handler: ~p~n",
+                [id_for_log(Id), Action, Reason]
+            )
+    end,
+    ok.
 
 id_for_log(console) -> "log.console_handler";
 id_for_log(Other) -> "log.file_handlers." ++ atom_to_list(Other).