From e7fc75fdf25686164d49d4ad8559a5db2d3506ee Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 27 Apr 2021 13:55:33 +0200 Subject: [PATCH 1/2] perf(broker): Optimization for handling bursty traffic intro. new lock type: 'spawn' of broker.perf.route_lock_type mnesia get lock calls are not optimized for selective receive. hence taking locks would be very expensive while there are tones of messages in the brokers message queue. This optimization run the transaction in a separate process to utilize the selective receive optimization of the compiler. --- priv/emqx.schema | 4 +++- src/emqx_router.erl | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index e759a3950..9d8470c88 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2259,9 +2259,11 @@ end}. %% key: mnesia translational updates with per-key locks. recommended for single node setup. %% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. %% global: global lock protected updates. recommended for larger cluster. +%% spawn: same as `key', but transaction is done in another proc, ideal for handling bursty traffic. +%% {mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ {default, key}, - {datatype, {enum, [key, tab, global]}} + {datatype, {enum, [key, tab, global, spawn]}} ]}. %% @doc Enable trie path compaction. diff --git a/src/emqx_router.erl b/src/emqx_router.erl index d731e2720..493527d13 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -263,7 +263,25 @@ maybe_trans(Fun, Args) -> trans(fun() -> emqx_trie:lock_tables(), apply(Fun, Args) - end, []) + end, []); + spawn -> + %% trigger selective receive optimization of compiler, + %% ideal for handling busty traffic. + Ref = erlang:make_ref(), + Owner = self(), + {WPid, RefMon} = spawn_monitor(fun() -> + Res = trans(Fun, Args), + Owner ! {Ref, Res} + end), + receive + {Ref, TransRes} -> + receive + {'DOWN', RefMon, process, WPid, normal} -> ok + end, + TransRes; + {'DOWN', RefMon, process, WPid, _Info} -> + {error, trans_crash} + end end. -spec(trans(function(), list(any())) -> ok | {error, term()}). From e122ac57165348bdbdeeaa01645e645b6ef30c8a Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 28 Apr 2021 23:26:11 +0200 Subject: [PATCH 2/2] perf(broker): speedup trans when broker has a big mqueue --- etc/emqx.conf | 4 ++-- priv/emqx.schema | 8 ++++---- src/emqx_router.erl | 43 +++++++++++++++++++++---------------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 692725314..f45e0eaa0 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2268,8 +2268,8 @@ broker.shared_dispatch_ack_enabled = false ## Value: Flag broker.route_batch_clean = off -## Performance toggle for subscribe/unsubscribe wildcard topic -## change this toggle only when there are many wildcard topics. +## Performance toggle for subscribe/unsubscribe wildcard topic. +## Change this toggle only when there are many wildcard topics. ## Value: Enum ## - key: mnesia translational updates with per-key locks. recommended for single node setup. ## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. diff --git a/priv/emqx.schema b/priv/emqx.schema index 9d8470c88..9503d0f66 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2254,16 +2254,16 @@ end}. {datatype, flag} ]}. -%% @doc performance toggle for subscribe/unsubscribe wildcard topic -%% change this toggle only when there are many wildcard topics. +%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. +%% Change this toggle only when there are many wildcard topics. %% key: mnesia translational updates with per-key locks. recommended for single node setup. %% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. %% global: global lock protected updates. recommended for larger cluster. -%% spawn: same as `key', but transaction is done in another proc, ideal for handling bursty traffic. +%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster %% {mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ {default, key}, - {datatype, {enum, [key, tab, global, spawn]}} + {datatype, {enum, [key, tab, global]}} ]}. %% @doc Enable trie path compaction. diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 493527d13..7288d2e19 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -263,32 +263,31 @@ maybe_trans(Fun, Args) -> trans(fun() -> emqx_trie:lock_tables(), apply(Fun, Args) - end, []); - spawn -> - %% trigger selective receive optimization of compiler, - %% ideal for handling busty traffic. - Ref = erlang:make_ref(), - Owner = self(), - {WPid, RefMon} = spawn_monitor(fun() -> - Res = trans(Fun, Args), - Owner ! {Ref, Res} - end), - receive - {Ref, TransRes} -> - receive - {'DOWN', RefMon, process, WPid, normal} -> ok - end, - TransRes; - {'DOWN', RefMon, process, WPid, _Info} -> - {error, trans_crash} - end + end, []) end. -spec(trans(function(), list(any())) -> ok | {error, term()}). trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of - {atomic, Ok} -> Ok; - {aborted, Reason} -> {error, Reason} + %% trigger selective receive optimization of compiler, + %% ideal for handling bursty traffic. + Ref = erlang:make_ref(), + Owner = self(), + {WPid, RefMon} = spawn_monitor( + fun() -> + Res = case mnesia:transaction(Fun, Args) of + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} + end, + Owner ! {Ref, Res} + end), + receive + {Ref, TransRes} -> + receive + {'DOWN', RefMon, process, WPid, normal} -> ok + end, + TransRes; + {'DOWN', RefMon, process, WPid, Info} -> + {error, {trans_crash, Info}} end. lock_router() ->