From 6f00f5145718776adbe974ce28f4d4e3a9682978 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:30:26 +0200 Subject: [PATCH 1/4] chore(ekka): Bump version --- CHANGES-5.0.md | 1 + apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 32ab74076..6fe656bde 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -32,6 +32,7 @@ ## Enhancements * Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) +* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.2 diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 888ddb15d..0c0a09cc0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 07362581c..55d3529f1 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.13.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, diff --git a/rebar.config b/rebar.config index d7d168f7f..7d2570636 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} From f5d63c1555ef2685972ddb931aa0d0a476044d58 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 18 Jul 2022 15:11:25 +0200 Subject: [PATCH 2/4] fix(conf): Wait for the cluster RPC shard --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 ++ apps/emqx_conf/src/emqx_conf.app.src | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 2478a2540..01d20bb34 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -262,6 +262,8 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> + %% Workaround for https://github.com/emqx/mria/issues/94: + mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index a4946e8cf..1441a4180 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, From 1ef898d126b1076058c5edbcc54eba38012ee4ba Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 18 Jul 2022 17:11:37 -0300 Subject: [PATCH 3/4] ci(rocksdb): conditionally load mnesia_rocksdb in release Otherwise, it might not be included in the release, since it's no longer declared as a dependency on the mria app. --- mix.exs | 72 +++++++++++++++++++++++++++++------------------- rebar.config.erl | 61 ++++++++++++++++++++++------------------ 2 files changed, 78 insertions(+), 55 deletions(-) diff --git a/mix.exs b/mix.exs index 55d3529f1..614fcfdba 100644 --- a/mix.exs +++ b/mix.exs @@ -170,35 +170,40 @@ defmodule EMQXUmbrella.MixProject do hocon: :load, emqx: :load, emqx_conf: :load, - emqx_machine: :permanent, - mria: :load, - mnesia: :load, - ekka: :load, - emqx_plugin_libs: :load, - esasl: :load, - observer_cli: :permanent, - system_monitor: :load, - emqx_http_lib: :permanent, - emqx_resource: :permanent, - emqx_connector: :permanent, - emqx_authn: :permanent, - emqx_authz: :permanent, - emqx_auto_subscribe: :permanent, - emqx_gateway: :permanent, - emqx_exhook: :permanent, - emqx_bridge: :permanent, - emqx_rule_engine: :permanent, - emqx_modules: :permanent, - emqx_management: :permanent, - emqx_dashboard: :permanent, - emqx_retainer: :permanent, - emqx_statsd: :permanent, - emqx_prometheus: :permanent, - emqx_psk: :permanent, - emqx_slow_subs: :permanent, - emqx_plugins: :permanent, - emqx_mix: :none + emqx_machine: :permanent ] ++ + if(enable_rocksdb?(), + do: [mnesia_rocksdb: :load], + else: [] + ) ++ + [ + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + esasl: :load, + observer_cli: :permanent, + system_monitor: :load, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_retainer: :permanent, + emqx_statsd: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent, + emqx_mix: :none + ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ if(enable_jq?(), do: [jq: :permanent], else: []) ++ @@ -616,6 +621,11 @@ defmodule EMQXUmbrella.MixProject do ]) or "1" == System.get_env("BUILD_WITH_QUIC") end + defp enable_rocksdb?() do + not build_without_rocksdb?() or + "1" == System.get_env("BUILD_WITH_QUIC") + end + defp pkg_vsn() do %{edition_type: edition_type} = check_profile!() basedir = Path.dirname(__ENV__.file) @@ -657,6 +667,12 @@ defmodule EMQXUmbrella.MixProject do String.downcase(opt) != "false" end + defp build_without_rocksdb?() do + opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false") + + String.downcase(opt) != "false" + end + defp from_rebar_to_eex_template(str) do # we must not consider surrounding space in the template var name # because some help strings contain informative variables that diff --git a/rebar.config.erl b/rebar.config.erl index 8b42d3ce3..d4adfaabc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -88,6 +88,10 @@ is_quicer_supported() -> is_win32() orelse is_centos_6()) orelse "1" == os:getenv("BUILD_WITH_QUIC"). +is_rocksdb_supported() -> + not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB")) orelse + "1" == os:getenv("BUILD_WITH_ROCKSDB"). + is_macos() -> {unix, darwin} =:= os:type(). @@ -318,34 +322,37 @@ relx_apps(ReleaseType, Edition) -> % started by emqx_machine {emqx, load}, {emqx_conf, load}, - emqx_machine, - {mnesia, load}, - {ekka, load}, - {emqx_plugin_libs, load}, - {esasl, load}, - observer_cli, - % started by emqx_machine - {system_monitor, load}, - emqx_http_lib, - emqx_resource, - emqx_connector, - emqx_authn, - emqx_authz, - emqx_auto_subscribe, - emqx_gateway, - emqx_exhook, - emqx_bridge, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard, - emqx_retainer, - emqx_statsd, - emqx_prometheus, - emqx_psk, - emqx_slow_subs, - emqx_plugins + emqx_machine ] ++ + [{mnesia_rocksdb, load} || is_rocksdb_supported()] ++ + [ + {mnesia, load}, + {ekka, load}, + {emqx_plugin_libs, load}, + {esasl, load}, + observer_cli, + % started by emqx_machine + {system_monitor, load}, + emqx_http_lib, + emqx_resource, + emqx_connector, + emqx_authn, + emqx_authz, + emqx_auto_subscribe, + emqx_gateway, + emqx_exhook, + emqx_bridge, + emqx_rule_engine, + emqx_modules, + emqx_management, + emqx_dashboard, + emqx_retainer, + emqx_statsd, + emqx_prometheus, + emqx_psk, + emqx_slow_subs, + emqx_plugins + ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ [jq || is_jq_supported()] ++ From a05b7bf555d4c6bd6e611cb2cff02dfae825e9a1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 19 Jul 2022 10:07:01 +0800 Subject: [PATCH 4/4] fix: dialyzer warning --- CHANGES-5.0.md | 8 ++++---- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 6fe656bde..fcf2154d5 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -19,6 +19,7 @@ * Improve the dashboard listener startup log, the listener name is no longer spliced with port information, and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) +* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.3 @@ -32,17 +33,16 @@ ## Enhancements * Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) -* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.2 -Announcemnet: EMQX team has decided to stop supporting relup for opensouce edition. -Going forward, it will be an enterprise only feature. +Announcement: EMQX team has decided to stop supporting relup for opensource edition. +Going forward, it will be an enterprise-only feature. Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions. For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain. -This had been the biggest obstacle for EMQX team to act agile enough in deliverying enhancements and fixes. +This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes. ## Enhancements diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 01d20bb34..6353a4efa 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -263,7 +263,7 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> %% Workaround for https://github.com/emqx/mria/issues/94: - mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), + _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs},