Merge pull request #11309 from thalesmg/fix-machine-boot-reboot-r51-20230719

fix(machine_boot): use shared list of reboot apps and add bridges to reboot list
2023-07-20 14:33:16 -03:00 · 2023-07-20 14:33:16 -03:00 · 54efc04a6f
parent 5240ba70d4 6cd503865b
commit 54efc04a6f
32 changed files with 275 additions and 309 deletions
--- a/2
+++ b/2
@ -295,7 +295,7 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))

 .PHONY: fmt
 fmt: $(REBAR)
-	@$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}'
+	@$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,priv,test}/**/*.{erl,hrl,app.src,eterm}'
 	@$(SCRIPTS)/erlfmt -w 'rebar.config.erl'
 	@mix format

--- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src
+++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        ecql
    ]},
    {env, []},
--- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src
+++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        clickhouse
    ]},
    {env, []},
--- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src
+++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        erlcloud
    ]},
    {env, []},
--- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src
+++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge_http,
        ehttpc
    ]},
    {env, []},
--- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl
+++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl
@ -598,6 +598,7 @@ start_cluster(Cluster) ->
        end,
        Cluster
    ),
+    NumNodes = length(Nodes),
    on_exit(fun() ->
        emqx_utils:pmap(
            fun(N) ->
@ -607,6 +608,11 @@ start_cluster(Cluster) ->
            Nodes
        )
    end),
+    {ok, _} = snabbkaffe:block_until(
+        %% -1 because only those that join the first node will emit the event.
+        ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}),
+        30_000
+    ),
    Nodes.

 wait_for_cluster_rpc(Node) ->
--- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src
+++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        hstreamdb_erl
    ]},
    {env, []},
--- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src
+++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src
@ -2,7 +2,7 @@
    {description, "EMQX HTTP Bridge and Connector Application"},
    {vsn, "0.1.1"},
    {registered, []},
-    {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, ehttpc]},
+    {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]},
    {env, []},
    {modules, []},
    {links, []}
--- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src
+++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        influxdb
    ]},
    {env, []},
--- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src
+++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src
@ -11,7 +11,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge_http,
        %% for module emqx_connector_http
        emqx_connector
    ]},
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
@ -7,7 +7,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        telemetry,
        wolff,
        brod,
--- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl
+++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl
@ -1071,13 +1071,14 @@ cluster(Config) ->
    Cluster = emqx_common_test_helpers:emqx_cluster(
        [core, core],
        [
-            {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]},
+            {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_kafka, emqx_bridge]},
            {listener_ports, []},
            {peer_mod, PeerModule},
            {priv_data_dir, PrivDataDir},
            {load_schema, true},
            {start_autocluster, true},
            {schema_mod, emqx_enterprise_schema},
+            {load_apps, [emqx_machine]},
            {env_handler, fun
                (emqx) ->
                    application:set_env(emqx, boot_modules, [broker, router]),
@ -1901,6 +1902,7 @@ t_cluster_node_down(Config) ->
    ?check_trace(
        begin
            {_N2, Opts2} = lists:nth(2, Cluster),
+            NumNodes = length(Cluster),
            Nodes =
                [N1, N2 | _] =
                lists:map(
@ -1925,6 +1927,11 @@ t_cluster_node_down(Config) ->
                15_000
            ),
            wait_for_cluster_rpc(N2),
+            {ok, _} = snabbkaffe:block_until(
+                %% -1 because only those that join the first node will emit the event.
+                ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}),
+                30_000
+            ),
            erpc:call(N2, fun() -> {ok, _} = create_bridge(Config) end),
            {ok, _} = snabbkaffe:receive_events(SRef0),
            lists:foreach(
@ -1980,7 +1987,7 @@ t_cluster_node_down(Config) ->
            ?assertEqual(NPartitions, map_size(Assignments)),
            NumPublished = ets:info(TId, size),
            %% All published messages are eventually received.
-            Published = receive_published(#{n => NumPublished, timeout => 3_000}),
+            Published = receive_published(#{n => NumPublished, timeout => 10_000}),
            ct:pal("published:\n  ~p", [Published]),
            ok
        end
--- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src
+++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src
@ -5,8 +5,7 @@
    {applications, [
        kernel,
        stdlib,
-        emqx_resource,
-        emqx_bridge
+        emqx_resource
    ]},
    {env, []},
    {modules, []},
--- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src
+++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src
@ -7,7 +7,6 @@
        stdlib,
        emqx_connector,
        emqx_resource,
-        emqx_bridge,
        emqx_mongodb
    ]},
    {env, []},
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
@ -1,14 +1,13 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_mqtt, [
    {description, "EMQX MQTT Broker Bridge"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
    {registered, []},
    {applications, [
        kernel,
        stdlib,
        emqx,
        emqx_resource,
-        emqx_bridge,
        emqtt
    ]},
    {env, []},
--- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src
+++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src
@ -7,7 +7,6 @@
        stdlib,
        emqx_connector,
        emqx_resource,
-        emqx_bridge,
        emqx_mysql
    ]},
    {env, []},
--- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src
+++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        opentsdb
    ]},
    {env, []},
--- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src
+++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        emqx_oracle
    ]},
    {env, []},
--- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src
+++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src
@ -5,8 +5,7 @@
    {applications, [
        kernel,
        stdlib,
-        emqx_resource,
-        emqx_bridge
+        emqx_resource
    ]},
    {env, []},
    {modules, []},
--- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src
+++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        pulsar
    ]},
    {env, []},
--- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl
+++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl
@ -547,6 +547,7 @@ start_cluster(Cluster) ->
            emqx_common_test_helpers:start_slave(Name, Opts)
         || {Name, Opts} <- Cluster
        ],
+    NumNodes = length(Nodes),
    on_exit(fun() ->
        emqx_utils:pmap(
            fun(N) ->
@ -556,6 +557,11 @@ start_cluster(Cluster) ->
            Nodes
        )
    end),
+    {ok, _} = snabbkaffe:block_until(
+        %% -1 because only those that join the first node will emit the event.
+        ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}),
+        30_000
+    ),
    Nodes.

 kill_resource_managers() ->
--- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src
+++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src
@ -6,8 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
-        ecql,
        rabbit_common,
        amqp_client
    ]},
--- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src
+++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src
@ -7,7 +7,6 @@
        stdlib,
        emqx_connector,
        emqx_resource,
-        emqx_bridge,
        emqx_redis
    ]},
    {env, []},
--- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src
+++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src
@ -2,7 +2,7 @@
    {description, "EMQX Enterprise RocketMQ Bridge"},
    {vsn, "0.1.3"},
    {registered, []},
-    {applications, [kernel, stdlib, emqx_resource, emqx_bridge, rocketmq]},
+    {applications, [kernel, stdlib, emqx_resource, rocketmq]},
    {env, []},
    {modules, []},
    {links, []}
--- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src
+++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src
@ -2,7 +2,7 @@
    {description, "EMQX Enterprise SQL Server Bridge"},
    {vsn, "0.1.2"},
    {registered, []},
-    {applications, [kernel, stdlib, emqx_resource, emqx_bridge, odbc]},
+    {applications, [kernel, stdlib, emqx_resource, odbc]},
    {env, []},
    {modules, []},
    {links, []}
--- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src
+++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src
@ -6,7 +6,6 @@
        kernel,
        stdlib,
        emqx_resource,
-        emqx_bridge,
        tdengine
    ]},
    {env, []},
--- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src
+++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src
@ -2,7 +2,7 @@
    {description, "EMQX Enterprise TimescaleDB Bridge"},
    {vsn, "0.1.2"},
    {registered, []},
-    {applications, [kernel, stdlib, emqx_resource, emqx_bridge]},
+    {applications, [kernel, stdlib, emqx_resource]},
    {env, []},
    {modules, []},
    {links, []}
--- a/apps/emqx_machine/priv/reboot_lists.eterm
+++ b/apps/emqx_machine/priv/reboot_lists.eterm
@ -0,0 +1,111 @@
+%% -*- mode: erlang; -*-
+#{
+    %% must always be of type `load'
+    db_apps =>
+        [
+            mnesia_rocksdb,
+            mnesia,
+            mria,
+            ekka
+        ],
+    system_apps =>
+        [
+            kernel,
+            sasl,
+            crypto,
+            public_key,
+            asn1,
+            syntax_tools,
+            ssl,
+            os_mon,
+            inets,
+            compiler,
+            runtime_tools,
+            redbug,
+            xmerl,
+            {hocon, load},
+            telemetry
+        ],
+    %% must always be of type `load'
+    common_business_apps =>
+        [
+            emqx,
+            emqx_conf,
+
+            esasl,
+            observer_cli,
+            tools,
+            covertool,
+            %% started by emqx_machine
+            system_monitor,
+            emqx_utils,
+            emqx_durable_storage,
+            emqx_http_lib,
+            emqx_resource,
+            emqx_connector,
+            emqx_authn,
+            emqx_authz,
+            emqx_auto_subscribe,
+            emqx_gateway,
+            emqx_gateway_stomp,
+            emqx_gateway_mqttsn,
+            emqx_gateway_coap,
+            emqx_gateway_lwm2m,
+            emqx_gateway_exproto,
+            emqx_exhook,
+            emqx_bridge,
+            emqx_bridge_mqtt,
+            emqx_bridge_http,
+            emqx_rule_engine,
+            emqx_modules,
+            emqx_management,
+            emqx_dashboard,
+            emqx_retainer,
+            emqx_prometheus,
+            emqx_psk,
+            emqx_slow_subs,
+            emqx_mongodb,
+            emqx_redis,
+            emqx_mysql,
+            emqx_plugins,
+            quicer,
+            bcrypt,
+            jq,
+            observer
+        ],
+    %% must always be of type `load'
+    ee_business_apps =>
+        [
+            emqx_license,
+            emqx_enterprise,
+            emqx_bridge_kafka,
+            emqx_bridge_pulsar,
+            emqx_bridge_gcp_pubsub,
+            emqx_bridge_cassandra,
+            emqx_bridge_opents,
+            emqx_bridge_clickhouse,
+            emqx_bridge_dynamo,
+            emqx_bridge_hstreamdb,
+            emqx_bridge_influxdb,
+            emqx_bridge_iotdb,
+            emqx_bridge_matrix,
+            emqx_bridge_mongodb,
+            emqx_bridge_mysql,
+            emqx_bridge_pgsql,
+            emqx_bridge_redis,
+            emqx_bridge_rocketmq,
+            emqx_bridge_tdengine,
+            emqx_bridge_timescale,
+            emqx_bridge_sqlserver,
+            emqx_oracle,
+            emqx_bridge_oracle,
+            emqx_bridge_rabbitmq,
+            emqx_schema_registry,
+            emqx_eviction_agent,
+            emqx_node_rebalance,
+            emqx_ft
+        ],
+    %% must always be of type `load'
+    ce_business_apps =>
+        [emqx_telemetry]
+}.
--- a/apps/emqx_machine/src/emqx_machine_boot.erl
+++ b/apps/emqx_machine/src/emqx_machine_boot.erl
@ -16,6 +16,7 @@
 -module(emqx_machine_boot).

 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").

 -export([post_boot/0]).
 -export([stop_apps/0, ensure_apps_started/0]).
@ -24,7 +25,6 @@
 -export([stop_port_apps/0]).

 -dialyzer({no_match, [basic_reboot_apps/0]}).
-dialyzer({no_match, [basic_reboot_apps_edition/1]}).

 -ifdef(TEST).
 -export([sorted_reboot_apps/1, reboot_apps/0]).
@ -94,7 +94,8 @@ stop_one_app(App) ->

 ensure_apps_started() ->
    ?SLOG(notice, #{msg => "(re)starting_emqx_apps"}),
-    lists:foreach(fun start_one_app/1, sorted_reboot_apps()).
+    lists:foreach(fun start_one_app/1, sorted_reboot_apps()),
+    ?tp(emqx_machine_boot_apps_started, #{}).

 start_one_app(App) ->
    ?SLOG(debug, #{msg => "starting_app", app => App}),
@ -128,44 +129,39 @@ reboot_apps() ->
    BaseRebootApps ++ ConfigApps.

 basic_reboot_apps() ->
-    ?BASIC_REBOOT_APPS ++
-        [
-            emqx_prometheus,
-            emqx_modules,
-            emqx_dashboard,
-            emqx_connector,
-            emqx_gateway,
-            emqx_resource,
-            emqx_rule_engine,
-            emqx_bridge,
-            emqx_management,
-            emqx_retainer,
-            emqx_exhook,
-            emqx_authn,
-            emqx_authz,
-            emqx_slow_subs,
-            emqx_auto_subscribe,
-            emqx_plugins,
-            emqx_psk
-        ] ++ basic_reboot_apps_edition(emqx_release:edition()).
+    PrivDir = code:priv_dir(emqx_machine),
+    RebootListPath = filename:join([PrivDir, "reboot_lists.eterm"]),
+    {ok, [
+        #{
+            common_business_apps := CommonBusinessApps,
+            ee_business_apps := EEBusinessApps,
+            ce_business_apps := CEBusinessApps
+        }
+    ]} = file:consult(RebootListPath),
+    EditionSpecificApps =
+        case emqx_release:edition() of
+            ee -> EEBusinessApps;
+            ce -> CEBusinessApps;
+            _ -> []
+        end,
+    BusinessApps = CommonBusinessApps ++ EditionSpecificApps,
+    ?BASIC_REBOOT_APPS ++ (BusinessApps -- excluded_apps()).

-basic_reboot_apps_edition(ce) ->
-    [emqx_telemetry];
-basic_reboot_apps_edition(ee) ->
-    [
-        emqx_license,
-        emqx_s3,
-        emqx_ft,
-        emqx_eviction_agent,
-        emqx_node_rebalance,
-        emqx_schema_registry
-    ];
-%% unexcepted edition, should not happen
-basic_reboot_apps_edition(_) ->
-    [].
+excluded_apps() ->
+    OptionalApps = [bcrypt, jq, observer],
+    [system_monitor, observer_cli] ++
+        [App || App <- OptionalApps, not is_app(App)].
+
+is_app(Name) ->
+    case application:load(Name) of
+        ok -> true;
+        {error, {already_loaded, _}} -> true;
+        _ -> false
+    end.

 sorted_reboot_apps() ->
-    Apps = [{App, app_deps(App)} || App <- reboot_apps()],
+    Apps0 = [{App, app_deps(App)} || App <- reboot_apps()],
+    Apps = inject_bridge_deps(Apps0),
    sorted_reboot_apps(Apps).

 app_deps(App) ->
@ -174,6 +170,25 @@ app_deps(App) ->
        {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List)
    end.

+%% `emqx_bridge' is special in that it needs all the bridges apps to
+%% be started before it, so that, when it loads the bridges from
+%% configuration, the bridge app and its dependencies need to be up.
+inject_bridge_deps(RebootAppDeps) ->
+    BridgeApps = [
+        App
+     || {App, _Deps} <- RebootAppDeps,
+        lists:prefix("emqx_bridge_", atom_to_list(App))
+    ],
+    lists:map(
+        fun
+            ({emqx_bridge, Deps0}) when is_list(Deps0) ->
+                {emqx_bridge, Deps0 ++ BridgeApps};
+            (App) ->
+                App
+        end,
+        RebootAppDeps
+    ).
+
 sorted_reboot_apps(Apps) ->
    G = digraph:new(),
    try
--- a/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl
+++ b/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl
@ -364,11 +364,7 @@ cluster(Config) ->
            {load_schema, true},
            {start_autocluster, true},
            {schema_mod, emqx_enterprise_schema},
-            %% need to restart schema registry app in the tests so
-            %% that it re-registers the config handler that is lost
-            %% when emqx_conf restarts during join.
-            {env, [{emqx_machine, applications, [emqx_schema_registry]}]},
-            {load_apps, [emqx_machine | ?APPS]},
+            {load_apps, [emqx_machine]},
            {env_handler, fun
                (emqx) ->
                    application:set_env(emqx, boot_modules, [broker, router]),
@ -388,6 +384,7 @@ start_cluster(Cluster) ->
        emqx_common_test_helpers:start_slave(Name, Opts)
     || {Name, Opts} <- Cluster
    ],
+    NumNodes = length(Nodes),
    on_exit(fun() ->
        emqx_utils:pmap(
            fun(N) ->
@ -397,7 +394,11 @@ start_cluster(Cluster) ->
            Nodes
        )
    end),
-    erpc:multicall(Nodes, mria_rlog, wait_for_shards, [[?SCHEMA_REGISTRY_SHARD], 30_000]),
+    {ok, _} = snabbkaffe:block_until(
+        %% -1 because only those that join the first node will emit the event.
+        ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}),
+        30_000
+    ),
    Nodes.

 wait_for_cluster_rpc(Node) ->
@ -658,7 +659,7 @@ t_cluster_serde_build(Config) ->
            Nodes = [N1, N2 | _] = start_cluster(Cluster),
            NumNodes = length(Nodes),
            wait_for_cluster_rpc(N2),
-            ?assertEqual(
+            ?assertMatch(
                ok,
                erpc:call(N2, emqx_schema_registry, add_schema, [SerdeName, Schema])
            ),
@ -687,7 +688,7 @@ t_cluster_serde_build(Config) ->
            {ok, SRef1} = snabbkaffe:subscribe(
                ?match_event(#{?snk_kind := schema_registry_serdes_deleted}),
                NumNodes,
-                5_000
+                10_000
            ),
            ?assertEqual(
                ok,
--- a/mix.exs
+++ b/mix.exs
@ -297,6 +297,7 @@ defmodule EMQXUmbrella.MixProject do
        [
          applications: applications(edition_type),
          skip_mode_validation_for: [
+            :emqx_mix,
            :emqx_gateway,
            :emqx_gateway_stomp,
            :emqx_gateway_mqttsn,
@ -316,7 +317,10 @@ defmodule EMQXUmbrella.MixProject do
            :emqx_auto_subscribe,
            :emqx_slow_subs,
            :emqx_plugins,
-            :emqx_ft
+            :emqx_ft,
+            :emqx_s3,
+            :emqx_durable_storage,
+            :rabbit_common
          ],
          steps: steps,
          strip_beams: false
@ -326,137 +330,54 @@ defmodule EMQXUmbrella.MixProject do
  end

  def applications(edition_type) do
-    system_apps = [
-      crypto: :permanent,
-      public_key: :permanent,
-      asn1: :permanent,
-      syntax_tools: :permanent,
-      ssl: :permanent,
-      os_mon: :permanent,
-      inets: :permanent,
-      compiler: :permanent,
-      runtime_tools: :permanent,
-      redbug: :permanent,
-      xmerl: :permanent,
-      hocon: :load,
-      telemetry: :permanent
-    ]
+    {:ok,
+     [
+       %{
+         db_apps: db_apps,
+         system_apps: system_apps,
+         common_business_apps: common_business_apps,
+         ee_business_apps: ee_business_apps,
+         ce_business_apps: ce_business_apps
+       }
+     ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")

-    db_apps =
-      if enable_rocksdb?() do
-        [:mnesia_rocksdb]
+    edition_specific_apps =
+      if edition_type == :enterprise do
+        ee_business_apps
      else
-        []
-      end ++
-        [
-          :mnesia,
-          :mria,
-          :ekka
-        ]
+        ce_business_apps
+      end

-    business_apps =
-      [
-        :emqx,
-        :emqx_conf,
-        :esasl,
-        :observer_cli,
-        :tools,
-        :covertool,
-        :system_monitor,
-        :emqx_utils,
-        :emqx_http_lib,
-        :emqx_resource,
-        :emqx_connector,
-        :emqx_authn,
-        :emqx_authz,
-        :emqx_auto_subscribe,
-        :emqx_gateway,
-        :emqx_gateway_stomp,
-        :emqx_gateway_mqttsn,
-        :emqx_gateway_coap,
-        :emqx_gateway_lwm2m,
-        :emqx_gateway_exproto,
-        :emqx_exhook,
-        :emqx_bridge,
-        :emqx_bridge_mqtt,
-        :emqx_bridge_http,
-        :emqx_rule_engine,
-        :emqx_modules,
-        :emqx_management,
-        :emqx_dashboard,
-        :emqx_retainer,
-        :emqx_prometheus,
-        :emqx_psk,
-        :emqx_slow_subs,
-        :emqx_mongodb,
-        :emqx_redis,
-        :emqx_mysql,
-        :emqx_plugins,
-        :emqx_mix
-      ] ++
-        if enable_quicer?() do
-          [:quicer]
-        else
-          []
-        end ++
-        if enable_bcrypt?() do
-          [:bcrypt]
-        else
-          []
-        end ++
-        if enable_jq?() do
-          [:jq]
-        else
-          []
-        end ++
-        if(is_app(:observer),
-          do: [:observer],
-          else: []
-        ) ++
-        case edition_type do
-          :enterprise ->
-            [
-              :emqx_license,
-              :emqx_enterprise,
-              :emqx_bridge_kafka,
-              :emqx_bridge_pulsar,
-              :emqx_bridge_gcp_pubsub,
-              :emqx_bridge_cassandra,
-              :emqx_bridge_opents,
-              :emqx_bridge_clickhouse,
-              :emqx_bridge_dynamo,
-              :emqx_bridge_hstreamdb,
-              :emqx_bridge_influxdb,
-              :emqx_bridge_iotdb,
-              :emqx_bridge_matrix,
-              :emqx_bridge_mongodb,
-              :emqx_bridge_mysql,
-              :emqx_bridge_pgsql,
-              :emqx_bridge_redis,
-              :emqx_bridge_rocketmq,
-              :emqx_bridge_tdengine,
-              :emqx_bridge_timescale,
-              :emqx_bridge_sqlserver,
-              :emqx_oracle,
-              :emqx_bridge_oracle,
-              :emqx_bridge_rabbitmq,
-              :emqx_schema_registry,
-              :emqx_eviction_agent,
-              :emqx_node_rebalance,
-              :emqx_ft
-            ]
+    business_apps = common_business_apps ++ edition_specific_apps

-          _ ->
-            [:emqx_telemetry]
-        end
+    excluded_apps = excluded_apps()

-    system_apps ++
-      Enum.map(db_apps, &{&1, :load}) ++
-      [emqx_machine: :permanent] ++
-      Enum.map(business_apps, &{&1, :load})
+    system_apps =
+      Enum.map(system_apps, fn app ->
+        if is_atom(app), do: {app, :permanent}, else: app
+      end)
+
+    db_apps = Enum.map(db_apps, &{&1, :load})
+    business_apps = Enum.map(business_apps, &{&1, :load})
+
+    [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
+    |> List.flatten()
+    |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  end

-  defp is_app(name) do
+  defp excluded_apps() do
+    %{
+      mnesia_rocksdb: enable_rocksdb?(),
+      quicer: enable_quicer?(),
+      bcrypt: enable_bcrypt?(),
+      jq: enable_jq?(),
+      observer: is_app?(:observer)
+    }
+    |> Enum.reject(&elem(&1, 1))
+    |> Enum.map(&elem(&1, 0))
+  end
+
+  defp is_app?(name) do
    case Application.load(name) do
      :ok ->
        true
--- a/rebar.config.erl
+++ b/rebar.config.erl
@ -386,84 +386,37 @@ overlay_vars_pkg(pkg) ->
    ].

 relx_apps(ReleaseType, Edition) ->
-    SystemApps =
-        [
-            kernel,
-            sasl,
-            crypto,
-            public_key,
-            asn1,
-            syntax_tools,
-            ssl,
-            os_mon,
-            inets,
-            compiler,
-            runtime_tools,
-            redbug,
-            xmerl,
-            {hocon, load},
-            telemetry
-        ],
-    DBApps =
-        [mnesia_rocksdb || is_rocksdb_supported()] ++
-            [
-                mnesia,
-                mria,
-                ekka
-            ],
-    BusinessApps =
-        [
-            emqx,
-            emqx_conf,
-
-            esasl,
-            observer_cli,
-            tools,
-            covertool,
-            % started by emqx_machine
-            system_monitor,
-            emqx_utils,
-            emqx_http_lib,
-            emqx_resource,
-            emqx_connector,
-            emqx_authn,
-            emqx_authz,
-            emqx_auto_subscribe,
-            emqx_gateway,
-            emqx_gateway_stomp,
-            emqx_gateway_mqttsn,
-            emqx_gateway_coap,
-            emqx_gateway_lwm2m,
-            emqx_gateway_exproto,
-            emqx_exhook,
-            emqx_bridge,
-            emqx_bridge_mqtt,
-            emqx_bridge_http,
-            emqx_rule_engine,
-            emqx_modules,
-            emqx_management,
-            emqx_dashboard,
-            emqx_retainer,
-            emqx_prometheus,
-            emqx_psk,
-            emqx_slow_subs,
-            emqx_mongodb,
-            emqx_redis,
-            emqx_mysql,
-            emqx_plugins
-        ] ++
-            [quicer || is_quicer_supported()] ++
-            [bcrypt || provide_bcrypt_release(ReleaseType)] ++
-            %% Started automatically when needed (only needs to be started when the
-            %% port implementation is used)
-            [jq || is_jq_supported()] ++
-            [observer || is_app(observer)] ++
-            relx_apps_per_edition(Edition),
+    {ok, [
+        #{
+            db_apps := DBApps,
+            system_apps := SystemApps,
+            common_business_apps := CommonBusinessApps,
+            ee_business_apps := EEBusinessApps,
+            ce_business_apps := CEBusinessApps
+        }
+    ]} = file:consult("apps/emqx_machine/priv/reboot_lists.eterm"),
+    EditionSpecificApps =
+        case Edition of
+            ee -> EEBusinessApps;
+            ce -> CEBusinessApps
+        end,
+    BusinessApps = CommonBusinessApps ++ EditionSpecificApps,
+    ExcludedApps = excluded_apps(ReleaseType),
    SystemApps ++
        %% EMQX starts the DB and the business applications:
-        [{App, load} || App <- DBApps] ++
+        [{App, load} || App <- (DBApps -- ExcludedApps)] ++
        [emqx_machine] ++
-        [{App, load} || App <- BusinessApps].
+        [{App, load} || App <- (BusinessApps -- ExcludedApps)].
+
+excluded_apps(ReleaseType) ->
+    OptionalApps = [
+        {quicer, is_quicer_supported()},
+        {bcrypt, provide_bcrypt_release(ReleaseType)},
+        {jq, is_jq_supported()},
+        {observer, is_app(observer)},
+        {mnesia_rocksdb, is_rocksdb_supported()}
+    ],
+    [App || {App, false} <- OptionalApps].

 is_app(Name) ->
    case application:load(Name) of
@ -472,40 +425,6 @@ is_app(Name) ->
        _ -> false
    end.

-relx_apps_per_edition(ee) ->
-    [
-        emqx_license,
-        emqx_enterprise,
-        emqx_bridge_kafka,
-        emqx_bridge_pulsar,
-        emqx_bridge_gcp_pubsub,
-        emqx_bridge_cassandra,
-        emqx_bridge_opents,
-        emqx_bridge_clickhouse,
-        emqx_bridge_dynamo,
-        emqx_bridge_hstreamdb,
-        emqx_bridge_influxdb,
-        emqx_bridge_iotdb,
-        emqx_bridge_matrix,
-        emqx_bridge_mongodb,
-        emqx_bridge_mysql,
-        emqx_bridge_pgsql,
-        emqx_bridge_redis,
-        emqx_bridge_rocketmq,
-        emqx_bridge_tdengine,
-        emqx_bridge_timescale,
-        emqx_bridge_sqlserver,
-        emqx_oracle,
-        emqx_bridge_oracle,
-        emqx_bridge_rabbitmq,
-        emqx_schema_registry,
-        emqx_eviction_agent,
-        emqx_node_rebalance,
-        emqx_ft
-    ];
-relx_apps_per_edition(ce) ->
-    [emqx_telemetry].
-
 relx_overlay(ReleaseType, Edition) ->
    [
        {mkdir, "log/"},