From af015bc2bb5e917824101e7bf671ec28243bdd41 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Sep 2023 21:57:19 +0300 Subject: [PATCH 1/3] chore(ekka): bump version to 0.15.13 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a404e5d81..b64c2181e 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -28,7 +28,7 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.11"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.13"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, diff --git a/mix.exs b/mix.exs index 29596c872..749258f3e 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.11", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.13", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.11", override: true}, diff --git a/rebar.config b/rebar.config index a8cc269e6..13afa2df3 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.11"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.13"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.11"}}} From e1456c1c0434413cbc5276d2eaaee217122e4ea8 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 12 Sep 2023 22:28:46 +0300 Subject: [PATCH 2/3] feat(emqx_conf_schema): add node.default_bootstrap_batch_size config parameter --- apps/emqx_conf/src/emqx_conf_schema.erl | 10 ++++++++++ rel/i18n/emqx_conf_schema.hocon | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 94cdd2ecd..c5ae33f1d 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -674,6 +674,16 @@ fields("node") -> default => #{} } )}, + {"default_bootstrap_batch_size", + sc( + pos_integer(), + #{ + mapping => "mria.bootstrap_batch_size", + importance => ?IMPORTANCE_HIDDEN, + default => 500, + desc => ?DESC(db_default_bootstrap_batch_size) + } + )}, {"broker_pool_size", sc( pos_integer(), diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 442df1fa8..547acd607 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -776,6 +776,16 @@ the default is to use the value set in db.default_shard_transport." db_shard_transports.label: """Shard Transports""" +db_default_bootstrap_batch_size.desc: +"""The number of built-in database table records to be pushed in one chunk from a core node to a replicant node during bootstrap. +The bootstrapping occurs when a new replicant node is added to the cluster or an existing replicant node reconnects to a core one. +Increasing this value may greatly reduce a replicant node startup time, especially when EMQX cluster interconnect network latency is high +and EMQX built-in database holds large amount of data, e.g. when the number of subscriptions is high. +Defaults to 500.""" + +db_default_bootstrap_batch_size.label: +"""Default Bootstrap Batch Size""" + node_broker_pool_size.desc: """The number of workers in emqx_broker pool. Increasing this value may improve performance by enhancing parallelism, especially when EMQX cluster interconnect network latency is high. From a336300ab5f90736dc15dea4a8bb3622b6b85843 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 13 Sep 2023 10:54:50 +0300 Subject: [PATCH 3/3] chore: add changelog --- changes/ce/feat-11597.en.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changes/ce/feat-11597.en.md diff --git a/changes/ce/feat-11597.en.md b/changes/ce/feat-11597.en.md new file mode 100644 index 000000000..46ea769c1 --- /dev/null +++ b/changes/ce/feat-11597.en.md @@ -0,0 +1,7 @@ +- Upgrade ekka to 0.15.13, which includes the following changes: + - upgrade Mria to 0.6.2: + - make bootstrap batch size configurable, [Mria PR](https://github.com/emqx/mria/pull/159) + - improve robustness of mria_membership processes, [Mria PR](https://github.com/emqx/mria/pull/156) + - fix log message formatting error +- Add `node.default_bootstrap_batch_size` option to EMQX configuration. +Increasing the value of this option can greatly reduce a replicant node startup time, especially when EMQX cluster interconnect network latency is high and EMQX built-in database holds large amount of data, e.g. when the number of subscriptions is high.