diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl
index c3bfe7912..793a96cf2 100644
--- a/apps/emqx/src/emqx_authentication.erl
+++ b/apps/emqx/src/emqx_authentication.erl
@@ -557,8 +557,14 @@ handle_info(Info, State) ->
     {noreply, State}.
 
 terminate(Reason, _State) ->
-    ?SLOG(error, #{msg => "emqx_authentication_terminating",
-                   reason => Reason}),
+    case Reason of
+        normal -> ok;
+        {shutdown, _} -> ok;
+        Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating",
+                                reason => Other})
+    end,
+    emqx_config_handler:remove_handler([authentication]),
+    emqx_config_handler:remove_handler([listeners, '?', '?', authentication]),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl
index 86182143a..2bea0cbe9 100644
--- a/apps/emqx/src/emqx_authentication_config.erl
+++ b/apps/emqx/src/emqx_authentication_config.erl
@@ -221,12 +221,12 @@ convert_certs(CertsDir, NewConfig, OldConfig) ->
             OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}),
             Diff = diff_certs(NewSSLOpts, OldSSLOpts),
             NSSLOpts = lists:foldl(fun({identical, K}, Acc) ->
-                                    Acc#{K => maps:get(K, OldSSLOpts)};
-                                    ({_, K}, Acc) ->
-                                    CertFile = generate_filename(CertsDir, K),
-                                    ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)),
-                                    Acc#{K => CertFile}
-                                end, NewSSLOpts, Diff),
+                                           Acc#{K => maps:get(K, OldSSLOpts)};
+                                      ({_, K}, Acc) ->
+                                           CertFile = generate_filename(CertsDir, K),
+                                           ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)),
+                                           Acc#{K => CertFile}
+                                   end, NewSSLOpts, Diff),
             NewConfig#{<<"ssl">> => NSSLOpts}
     end.
 
diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl
index 297f1147d..c44a0cb96 100644
--- a/apps/emqx/src/emqx_config_handler.erl
+++ b/apps/emqx/src/emqx_config_handler.erl
@@ -77,9 +77,10 @@ update_config(SchemaModule, ConfKeyPath, UpdateArgs) ->
 add_handler(ConfKeyPath, HandlerName) ->
     gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}, infinity).
 
+%% @doc Remove handler asynchronously
 -spec remove_handler(emqx_config:config_key_path()) -> ok.
 remove_handler(ConfKeyPath) ->
-    gen_server:call(?MODULE, {remove_handler, ConfKeyPath}, infinity).
+    gen_server:cast(?MODULE, {remove_handler, ConfKeyPath}).
 
 %%============================================================================
 
@@ -95,11 +96,6 @@ handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers :
             {reply, Error, State}
     end;
 
-handle_call({remove_handler, ConfKeyPath}, _From,
-            State = #{handlers := Handlers}) ->
-    {reply, ok, State#{handlers =>
-        emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}};
-
 handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From,
             #{handlers := Handlers} = State) ->
     Reply = try
@@ -125,6 +121,9 @@ handle_call(_Request, _From, State) ->
     Reply = ok,
     {reply, Reply, State}.
 
+handle_cast({remove_handler, ConfKeyPath},
+            State = #{handlers := Handlers}) ->
+    {noreply, State#{handlers => emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}};
 handle_cast(_Msg, State) ->
     {noreply, State}.