Merge pull request #9919 from terry-xiaoyu/fix-application-stop-order

fix: the application stop order is wrong after ekka:join/1

changes/v4.4.15-en.md

@@ -31,3 +31,5 @@
 - fix the emqx reports `{case_clause,{error,closed}}` error log message when websocket connections interrupted [emqx/cowboy#8](https://github.com/emqx/cowboy/pull/8).
 
 - fix sometimes the rules cannot be enabled automatically after emqx is restarted [#9911](https://github.com/emqx/emqx/pull/9911).
+
+- fix the `{badarg,[{ets,lookup,[gproc,{shared, ...` error logs during shutdown [#9919](https://github.com/emqx/emqx/pull/9919).

changes/v4.4.15-zh.md

@@ -30,3 +30,5 @@
 - 修复 Websocket 连接中断时日志报 `{case_clause,{error,closed}}` 错误的问题 [emqx/cowboy#8](https://github.com/emqx/cowboy/pull/8)。
 
 - 修复某些情况下，重启 emqx 后规则无法自动启用的问题 [#9911](https://github.com/emqx/emqx/pull/9911)。
+
+- 修复停止 emqx 的时候，日志出现 `{badarg,[{ets,lookup,[gproc,{shared, ...` 错误的问题 [#9919](https://github.com/emqx/emqx/pull/9919)。

src/emqx.appup.src

@@ -8,6 +8,7 @@
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]},
@@ -20,6 +21,7 @@
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.12",
@@ -31,6 +33,7 @@
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.11",
@@ -41,6 +44,7 @@
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {add_module,emqx_cover},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
@@ -467,6 +471,7 @@
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]},
@@ -479,6 +484,7 @@
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.12",
@@ -490,6 +496,7 @@
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.11",
@@ -499,6 +506,7 @@
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},

src/emqx.erl

@@ -65,6 +65,8 @@
 
 %% Troubleshooting
 -export([ set_debug_secret/1
+        , default_started_applications/0
+        , expand_apps/1
         ]).
 
 -define(APP, ?MODULE).
@@ -252,14 +254,6 @@ is_application_running(App) ->
     StartedApps = proplists:get_value(started, application:info()),
     proplists:is_defined(App, StartedApps).
 
--ifdef(EMQX_ENTERPRISE).
-default_started_applications() ->
-    [gproc, esockd, ranch, cowboy, ekka, emqx].
--else.
-default_started_applications() ->
-    [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules].
--endif.
-
 -ifdef(EMQX_ENTERPRISE).
 on_reboot() ->
     try
@@ -290,6 +284,54 @@ on_shutdown(_) ->
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
+-ifdef(EMQX_ENTERPRISE).
+applications_need_restart() ->
+    [gproc, esockd, ranch, cowboy, ekka, emqx].
+-else.
+applications_need_restart() ->
+    [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules].
+-endif.
+
+-define(PK_START_APPS, {?MODULE, default_started_applications}).
+default_started_applications() ->
+    case persistent_term:get(?PK_START_APPS, undefined) of
+        undefined ->
+            AppNames = expand_apps(applications_need_restart()),
+            ok = persistent_term:put(?PK_START_APPS, AppNames),
+            AppNames;
+        AppNames ->
+            AppNames
+    end.
+
+%% expand the application list with dependent apps.
+expand_apps(AppNames) ->
+    AllApps = application:which_applications(),
+    remove_duplicated(
+        lists:flatmap(fun(AppName) ->
+                expand_an_app(AppName, AllApps)
+            end, AppNames)).
+
+expand_an_app(AppNameA, AllApps) ->
+    expand_an_app(AppNameA, AllApps, [AppNameA]).
+
+expand_an_app(_AppNameA, [], Acc) ->
+    Acc;
+expand_an_app(AppNameA, [{AppNameB, _Descr, _Vsn} | AllApps], Acc) ->
+    {ok, DepAppNames} = application:get_key(AppNameB, applications),
+    case lists:member(AppNameA, DepAppNames) of
+        true -> %% AppNameB depends on AppNameA
+            NewAcc = Acc ++ expand_an_app(AppNameB, AllApps),
+            expand_an_app(AppNameA, AllApps, NewAcc);
+        false ->
+            expand_an_app(AppNameA, AllApps, Acc)
+    end.
+
+remove_duplicated([]) -> [];
+remove_duplicated([E | Elems]) ->
+    case lists:member(E, Elems) of
+        true -> remove_duplicated(Elems);
+        false -> [E] ++ remove_duplicated(Elems)
+    end.
 
 reload_config(ConfFile) ->
     {ok, [Conf]} = file:consult(ConfFile),