Merge pull request #11235 from zhongwencool/clean-config-loaded-when-stop

fix: unset config_loader in emqx's env when stop emqx app

apps/emqx/src/emqx_app.erl

@@ -25,7 +25,9 @@
     get_description/0,
     get_release/0,
     set_config_loader/1,
-    get_config_loader/0
+    get_config_loader/0,
+    unset_config_loaded/0,
+    init_load_done/0
 ]).
 
 -include("logger.hrl").
@@ -54,14 +56,22 @@ prep_stop(_State) ->
 
 stop(_State) -> ok.
 
+-define(CONFIG_LOADER, config_loader).
+-define(DEFAULT_LOADER, emqx).
 %% @doc Call this function to make emqx boot without loading config,
 %% in case we want to delegate the config load to a higher level app
 %% which manages emqx app.
 set_config_loader(Module) when is_atom(Module) ->
-    application:set_env(emqx, config_loader, Module).
+    application:set_env(emqx, ?CONFIG_LOADER, Module).
 
 get_config_loader() ->
-    application:get_env(emqx, config_loader, emqx).
+    application:get_env(emqx, ?CONFIG_LOADER, ?DEFAULT_LOADER).
+
+unset_config_loaded() ->
+    application:unset_env(emqx, ?CONFIG_LOADER).
+
+init_load_done() ->
+    get_config_loader() =/= ?DEFAULT_LOADER.
 
 maybe_load_config() ->
     case get_config_loader() of

apps/emqx/src/emqx_listeners.erl

@@ -96,7 +96,9 @@ format_list(Listener) ->
 
 do_list_raw() ->
     %% GET /listeners from other nodes returns [] when init config is not loaded.
-    case emqx_app:get_config_loader() =/= emqx of
+    %% FIXME This is a workaround for the issue:
+    %% mria:running_nodes() sometime return node which not ready to accept rpc call.
+    case emqx_app:init_load_done() of
         true ->
             Key = <<"listeners">>,
             Raw = emqx_config:get_raw([Key], #{}),

apps/emqx_conf/src/emqx_conf_app.erl

@@ -21,9 +21,7 @@
 -export([start/2, stop/1]).
 -export([get_override_config_file/0]).
 -export([sync_data_from_node/0]).
-
+-export([unset_config_loaded/0]).
-%% Test purposes
--export([init_load_done/0]).
 
 -include_lib("emqx/include/logger.hrl").
 -include("emqx_conf.hrl").
@@ -45,11 +43,16 @@ start(_StartType, _StartArgs) ->
 stop(_State) ->
     ok.
 
+%% @doc emqx_conf relies on this flag to synchronize configuration between nodes.
+%% Therefore, we must clean up this flag when emqx application is restarted by mria.
+unset_config_loaded() ->
+    emqx_app:unset_config_loaded().
+
 %% Read the cluster config from the local node.
 %% This function is named 'override' due to historical reasons.
 get_override_config_file() ->
     Node = node(),
-    case init_load_done() of
+    case emqx_app:init_load_done() of
         false ->
             {error, #{node => Node, msg => "init_conf_load_not_done"}};
         true ->
@@ -109,10 +112,6 @@ init_load(TnxId) ->
             })
     end.
 
-init_load_done() ->
-    % NOTE: Either us or some higher level (i.e. tests) code loaded config.
-    emqx_app:get_config_loader() =/= emqx.
-
 init_conf() ->
     emqx_cluster_rpc:wait_for_cluster_rpc(),
     {ok, TnxId} = sync_cluster_conf(),

apps/emqx_conf/test/emqx_conf_app_SUITE.erl

@@ -215,7 +215,7 @@ assert_no_cluster_conf_copied([Node | Nodes], File) ->
 assert_config_load_done(Nodes) ->
     lists:foreach(
         fun(Node) ->
-            Done = rpc:call(Node, emqx_conf_app, init_load_done, []),
+            Done = rpc:call(Node, emqx_app, init_load_done, []),
             ?assert(Done, #{node => Node})
         end,
         Nodes

apps/emqx_machine/src/emqx_machine.app.src

@@ -3,7 +3,7 @@
     {id, "emqx_machine"},
     {description, "The EMQX Machine"},
     % strict semver, bump manually!
-    {vsn, "0.2.7"},
+    {vsn, "0.2.8"},
     {modules, []},
     {registered, []},
     {applications, [kernel, stdlib, emqx_ctl]},

apps/emqx_machine/src/emqx_machine_boot.erl

@@ -61,6 +61,7 @@ start_autocluster() ->
 stop_apps() ->
     ?SLOG(notice, #{msg => "stopping_emqx_apps"}),
     _ = emqx_alarm_handler:unload(),
+    ok = emqx_conf_app:unset_config_loaded(),
     lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())).
 
 %% Those port apps are terminated after the main apps