From cee378c345b33d1a5ba306876d088bdd7147883a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Sun, 18 Apr 2021 23:24:16 +0200
Subject: [PATCH] perf(trie): use global lock

Use global lock to reduce remote lock overhead.

So that emqx route trans can run in dirty *sync* context.

At least 10X subscribe/unsubscribe improvments.
---
 src/emqx_router.erl | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/emqx_router.erl b/src/emqx_router.erl
index cc358758f..56e6d8abb 100644
--- a/src/emqx_router.erl
+++ b/src/emqx_router.erl
@@ -118,7 +118,12 @@ do_add_route(Topic, Dest) when is_binary(Topic) ->
         false ->
             ok = emqx_router_helper:monitor(Dest),
             case emqx_topic:wildcard(Topic) of
-                true  -> trans(fun insert_trie_route/1, [Route]);
+                true  ->
+                    lock_router(),
+                    try trans(fun insert_trie_route/1, [Route])
+                    after
+                        unlock_router()
+                    end;
                 false -> insert_direct_route(Route)
             end
     end.
@@ -164,7 +169,12 @@ do_delete_route(Topic) when is_binary(Topic) ->
 do_delete_route(Topic, Dest) ->
     Route = #route{topic = Topic, dest = Dest},
     case emqx_topic:wildcard(Topic) of
-        true  -> trans(fun delete_trie_route/1, [Route]);
+        true  ->
+            lock_router(),
+            try trans(fun delete_trie_route/1, [Route])
+            after
+                unlock_router()
+            end;
         false -> delete_direct_route(Route)
     end.
 
@@ -249,8 +259,19 @@ delete_trie_route(Route = #route{topic = Topic}) ->
 %% @private
 -spec(trans(function(), list(any())) -> ok | {error, term()}).
 trans(Fun, Args) ->
-    case mnesia:transaction(Fun, Args) of
-        {atomic, Ok} -> Ok;
-        {aborted, Reason} -> {error, Reason}
+    mnesia:sync_dirty(Fun, Args).
+
+lock_router() ->
+    %% if Retry is not 0, global:set_lock could sleep a random time up to 8s.
+    %% Considering we have a limited number of brokers, it is safe to use sleep 1 ms.
+    case global:set_lock({?MODULE, self()}, [node() | nodes()], 0) of
+        false ->
+            %% Force to sleep 1ms instead.
+            timer:sleep(1),
+            lock_router();
+        true ->
+            ok
     end.
 
+unlock_router() ->
+    global:del_lock({?MODULE, self()}).