diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl
index e2e95d7ad..3588f1c8e 100644
--- a/apps/emqx/test/emqx_authentication_SUITE.erl
+++ b/apps/emqx/test/emqx_authentication_SUITE.erl
@@ -128,6 +128,7 @@ all() ->
 
 init_per_suite(Config) ->
     application:set_env(ekka, strict_mode, true),
+    emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.
 
diff --git a/apps/emqx/test/emqx_boot_SUITE.erl b/apps/emqx/test/emqx_boot_SUITE.erl
index 7d0a7b4d3..06f08afb8 100644
--- a/apps/emqx/test/emqx_boot_SUITE.erl
+++ b/apps/emqx/test/emqx_boot_SUITE.erl
@@ -24,19 +24,23 @@
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 t_is_enabled(_) ->
-    ok = application:set_env(emqx, boot_modules, all),
-    ?assert(emqx_boot:is_enabled(router)),
-    ?assert(emqx_boot:is_enabled(broker)),
-    ?assert(emqx_boot:is_enabled(listeners)),
-    ok = application:set_env(emqx, boot_modules, [router]),
-    ?assert(emqx_boot:is_enabled(router)),
-    ?assertNot(emqx_boot:is_enabled(broker)),
-    ?assertNot(emqx_boot:is_enabled(listeners)),
-    ok = application:set_env(emqx, boot_modules, [router, broker]),
-    ?assert(emqx_boot:is_enabled(router)),
-    ?assert(emqx_boot:is_enabled(broker)),
-    ?assertNot(emqx_boot:is_enabled(listeners)),
-    ok = application:set_env(emqx, boot_modules, [router, broker, listeners]),
-    ?assert(emqx_boot:is_enabled(router)),
-    ?assert(emqx_boot:is_enabled(broker)),
-    ?assert(emqx_boot:is_enabled(listeners)).
+    try
+        ok = application:set_env(emqx, boot_modules, all),
+        ?assert(emqx_boot:is_enabled(router)),
+        ?assert(emqx_boot:is_enabled(broker)),
+        ?assert(emqx_boot:is_enabled(listeners)),
+        ok = application:set_env(emqx, boot_modules, [router]),
+        ?assert(emqx_boot:is_enabled(router)),
+        ?assertNot(emqx_boot:is_enabled(broker)),
+        ?assertNot(emqx_boot:is_enabled(listeners)),
+        ok = application:set_env(emqx, boot_modules, [router, broker]),
+        ?assert(emqx_boot:is_enabled(router)),
+        ?assert(emqx_boot:is_enabled(broker)),
+        ?assertNot(emqx_boot:is_enabled(listeners)),
+        ok = application:set_env(emqx, boot_modules, [router, broker, listeners]),
+        ?assert(emqx_boot:is_enabled(router)),
+        ?assert(emqx_boot:is_enabled(broker)),
+        ?assert(emqx_boot:is_enabled(listeners))
+    after
+        application:set_env(emqx, boot_modules, all)
+    end.
diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf
index 337823233..21943e036 100644
--- a/apps/emqx_conf/i18n/emqx_conf_schema.conf
+++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf
@@ -456,19 +456,6 @@ the old dir should be deleted first.<br/>
     }
   }
 
-  node_config_files {
-    desc {
-      en: """List of configuration files that are read during startup. The order is
-significant: later configuration files override the previous ones.
-          """
-      zh: """启动时读取的配置文件列表。后面的配置文件项覆盖前面的文件。"""
-    }
-    label {
-      en: "Config Files"
-      zh: "配置文件"
-    }
-  }
-
   node_global_gc_interval {
     desc {
       en: """Periodic garbage collection interval. Set to <code>disabled</code> to have it disabled."""
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 7a20a88dc..3cfb74b0e 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -466,9 +466,9 @@ fields("node") ->
                 hoconsc:array(string()),
                 #{
                     mapping => "emqx.config_files",
-                    default => undefined,
-                    'readOnly' => true,
-                    desc => ?DESC(node_config_files)
+                    hidden => true,
+                    required => false,
+                    'readOnly' => true
                 }
             )},
         {"global_gc_interval",
@@ -1037,17 +1037,13 @@ metrics_enabled(disabled) -> [].
 tr_default_config_driver(Conf) ->
     conf_get("rpc.driver", Conf).
 
-tr_config_files(Conf) ->
-    case conf_get("emqx.config_files", Conf) of
-        [_ | _] = Files ->
-            Files;
-        _ ->
-            case os:getenv("EMQX_ETC_DIR") of
-                false ->
-                    [filename:join([code:lib_dir(emqx), "etc", "emqx.conf"])];
-                Dir ->
-                    [filename:join([Dir, "emqx.conf"])]
-            end
+tr_config_files(_Conf) ->
+    case os:getenv("EMQX_ETC_DIR") of
+        false ->
+            %% testing, or running emqx app as deps
+            [filename:join([code:lib_dir(emqx), "etc", "emqx.conf"])];
+        Dir ->
+            [filename:join([Dir, "emqx.conf"])]
     end.
 
 tr_cluster_override_conf_file(Conf) ->
diff --git a/changes/v5.0.15/fix-9750.en.md b/changes/v5.0.15/fix-9750.en.md
new file mode 100644
index 000000000..98c07dfb6
--- /dev/null
+++ b/changes/v5.0.15/fix-9750.en.md
@@ -0,0 +1,5 @@
+Reload overriding configs after boot.
+Prior to this change, two configs were allow to change from dashboard, but will not take effect after reboot:
+  * Logging (such as level)
+  * Prometheus configs
+
diff --git a/changes/v5.0.15/fix-9750.zh.md b/changes/v5.0.15/fix-9750.zh.md
new file mode 100644
index 000000000..605d4c225
--- /dev/null
+++ b/changes/v5.0.15/fix-9750.zh.md
@@ -0,0 +1,4 @@
+启动后重新加载一些重载配置项。
+在此修复前，下面两个配置项允许从 Dashboard 控制台修改，但是在重启后无法生效：
+  * 日志 （例如日志级别）
+  * Prometheus 配置