Merge pull request #8502 from ieQu1/wait-cluster-rpc-shard

Wait cluster rpc shard

CHANGES-5.0.md

@@ -19,6 +19,7 @@
 * Improve the dashboard listener startup log, the listener name is no longer spliced with port information,
   and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480)
 * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485)
+* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439)
 
 # 5.0.3
 
@@ -35,13 +36,13 @@
 
 # 5.0.2
 
-Announcemnet: EMQX team has decided to stop supporting relup for opensouce edition.
+Announcement: EMQX team has decided to stop supporting relup for opensource edition.
-Going forward, it will be an enterprise only feature.
+Going forward, it will be an enterprise-only feature.
 
 Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions.
 
 For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain.
-This had been the biggest obstacle for EMQX team to act agile enough in deliverying enhancements and fixes.
+This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes.
 
 ## Enhancements
 

apps/emqx/rebar.config

@@ -27,7 +27,7 @@
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

apps/emqx_conf/src/emqx_cluster_rpc.erl

@@ -262,6 +262,8 @@ fast_forward_to_commit(Node, ToTnxId) ->
 
 %% @private
 init([Node, RetryMs]) ->
+    %% Workaround for https://github.com/emqx/mria/issues/94:
+    _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
     _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
     {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}),
     State = #{node => Node, retry_interval => RetryMs},

apps/emqx_conf/src/emqx_conf.app.src

@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},

mix.exs

@@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.13.1", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
@@ -170,35 +170,40 @@ defmodule EMQXUmbrella.MixProject do
       hocon: :load,
       emqx: :load,
       emqx_conf: :load,
-      emqx_machine: :permanent,
+      emqx_machine: :permanent
-      mria: :load,
-      mnesia: :load,
-      ekka: :load,
-      emqx_plugin_libs: :load,
-      esasl: :load,
-      observer_cli: :permanent,
-      system_monitor: :load,
-      emqx_http_lib: :permanent,
-      emqx_resource: :permanent,
-      emqx_connector: :permanent,
-      emqx_authn: :permanent,
-      emqx_authz: :permanent,
-      emqx_auto_subscribe: :permanent,
-      emqx_gateway: :permanent,
-      emqx_exhook: :permanent,
-      emqx_bridge: :permanent,
-      emqx_rule_engine: :permanent,
-      emqx_modules: :permanent,
-      emqx_management: :permanent,
-      emqx_dashboard: :permanent,
-      emqx_retainer: :permanent,
-      emqx_statsd: :permanent,
-      emqx_prometheus: :permanent,
-      emqx_psk: :permanent,
-      emqx_slow_subs: :permanent,
-      emqx_plugins: :permanent,
-      emqx_mix: :none
     ] ++
+      if(enable_rocksdb?(),
+        do: [mnesia_rocksdb: :load],
+        else: []
+      ) ++
+      [
+        mnesia: :load,
+        ekka: :load,
+        emqx_plugin_libs: :load,
+        esasl: :load,
+        observer_cli: :permanent,
+        system_monitor: :load,
+        emqx_http_lib: :permanent,
+        emqx_resource: :permanent,
+        emqx_connector: :permanent,
+        emqx_authn: :permanent,
+        emqx_authz: :permanent,
+        emqx_auto_subscribe: :permanent,
+        emqx_gateway: :permanent,
+        emqx_exhook: :permanent,
+        emqx_bridge: :permanent,
+        emqx_rule_engine: :permanent,
+        emqx_modules: :permanent,
+        emqx_management: :permanent,
+        emqx_dashboard: :permanent,
+        emqx_retainer: :permanent,
+        emqx_statsd: :permanent,
+        emqx_prometheus: :permanent,
+        emqx_psk: :permanent,
+        emqx_slow_subs: :permanent,
+        emqx_plugins: :permanent,
+        emqx_mix: :none
+      ] ++
       if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
       if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
       if(enable_jq?(), do: [jq: :permanent], else: []) ++
@@ -616,6 +621,11 @@ defmodule EMQXUmbrella.MixProject do
     ]) or "1" == System.get_env("BUILD_WITH_QUIC")
   end
 
+  defp enable_rocksdb?() do
+    not build_without_rocksdb?() or
+      "1" == System.get_env("BUILD_WITH_QUIC")
+  end
+
   defp pkg_vsn() do
     %{edition_type: edition_type} = check_profile!()
     basedir = Path.dirname(__ENV__.file)
@@ -657,6 +667,12 @@ defmodule EMQXUmbrella.MixProject do
     String.downcase(opt) != "false"
   end
 
+  defp build_without_rocksdb?() do
+    opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
+
+    String.downcase(opt) != "false"
+  end
+
   defp from_rebar_to_eex_template(str) do
     # we must not consider surrounding space in the template var name
     # because some help strings contain informative variables that

rebar.config

@@ -54,7 +54,7 @@
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}

rebar.config.erl

@@ -88,6 +88,10 @@ is_quicer_supported() ->
         is_win32() orelse is_centos_6()) orelse
         "1" == os:getenv("BUILD_WITH_QUIC").
 
+is_rocksdb_supported() ->
+    not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB")) orelse
+        "1" == os:getenv("BUILD_WITH_ROCKSDB").
+
 is_macos() ->
     {unix, darwin} =:= os:type().
 
@@ -318,34 +322,37 @@ relx_apps(ReleaseType, Edition) ->
         % started by emqx_machine
         {emqx, load},
         {emqx_conf, load},
-        emqx_machine,
+        emqx_machine
-        {mnesia, load},
-        {ekka, load},
-        {emqx_plugin_libs, load},
-        {esasl, load},
-        observer_cli,
-        % started by emqx_machine
-        {system_monitor, load},
-        emqx_http_lib,
-        emqx_resource,
-        emqx_connector,
-        emqx_authn,
-        emqx_authz,
-        emqx_auto_subscribe,
-        emqx_gateway,
-        emqx_exhook,
-        emqx_bridge,
-        emqx_rule_engine,
-        emqx_modules,
-        emqx_management,
-        emqx_dashboard,
-        emqx_retainer,
-        emqx_statsd,
-        emqx_prometheus,
-        emqx_psk,
-        emqx_slow_subs,
-        emqx_plugins
     ] ++
+        [{mnesia_rocksdb, load} || is_rocksdb_supported()] ++
+        [
+            {mnesia, load},
+            {ekka, load},
+            {emqx_plugin_libs, load},
+            {esasl, load},
+            observer_cli,
+            % started by emqx_machine
+            {system_monitor, load},
+            emqx_http_lib,
+            emqx_resource,
+            emqx_connector,
+            emqx_authn,
+            emqx_authz,
+            emqx_auto_subscribe,
+            emqx_gateway,
+            emqx_exhook,
+            emqx_bridge,
+            emqx_rule_engine,
+            emqx_modules,
+            emqx_management,
+            emqx_dashboard,
+            emqx_retainer,
+            emqx_statsd,
+            emqx_prometheus,
+            emqx_psk,
+            emqx_slow_subs,
+            emqx_plugins
+        ] ++
         [quicer || is_quicer_supported()] ++
         [bcrypt || provide_bcrypt_release(ReleaseType)] ++
         [jq || is_jq_supported()] ++