Merge emqx32

Makefile

@@ -18,7 +18,7 @@ NO_AUTOPATCH = cuttlefish
 ERLC_OPTS += +debug_info -DAPPLICATION=emqx
 
 BUILD_DEPS = cuttlefish
-dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.1.1
+dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.0
 
 #TEST_DEPS = emqx_ct_helplers
 #dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers
@@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
 			emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \
 			emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \
 			emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \
-			emqx_hooks emqx_batch
+			emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc
 
 CT_NODE_NAME = emqxct@127.0.0.1
 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME)

etc/emqx.conf

@@ -160,11 +160,6 @@ node.name = emqx@127.0.0.1
 ## Value: String
 node.cookie = emqxsecretcookie
 
-## Enable SMP support of Erlang VM.
-##
-## Value: enable | auto | disable
-node.smp = auto
-
 ## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable
 ## heartbeat, or set the value as 'on'
 ##
@@ -173,13 +168,6 @@ node.smp = auto
 ## vm.args: -heart
 ## node.heartbeat = on
 
-## Enable kernel poll.
-##
-## Value: on | off
-##
-## Default: on
-node.kernel_poll = on
-
 ## Sets the number of threads in async thread pool. Valid range is 0-1024.
 ##
 ## See: http://erlang.org/doc/man/erl.html
@@ -768,6 +756,11 @@ listener.tcp.external.max_connections = 1024000
 ## Value: Number
 listener.tcp.external.max_conn_rate = 1000
 
+## Specify the {active, N} option for the external MQTT/TCP Socket.
+##
+## Value: Number
+listener.tcp.external.active_n = 100
+
 ## Zone of the external MQTT/TCP listener belonged to.
 ##
 ## See: zone.$name.*
@@ -904,6 +897,11 @@ listener.tcp.internal.max_connections = 1024000
 ## Value: Number
 listener.tcp.internal.max_conn_rate = 1000
 
+## Specify the {active, N} option for the internal MQTT/TCP Socket.
+##
+## Value: Number
+listener.tcp.internal.active_n = 1000
+
 ## Zone of the internal MQTT/TCP listener belonged to.
 ##
 ## Value: String
@@ -1011,6 +1009,11 @@ listener.ssl.external.max_connections = 102400
 ## Value: Number
 listener.ssl.external.max_conn_rate = 500
 
+## Specify the {active, N} option for the internal MQTT/SSL Socket.
+##
+## Value: Number
+listener.ssl.external.active_n = 100
+
 ## Zone of the external MQTT/SSL listener belonged to.
 ##
 ## Value: String
@@ -1916,6 +1919,11 @@ plugins.expand_plugins_dir = {{ platform_plugins_dir }}/
 ## Default: 1m, 1 minute
 broker.sys_interval = 1m
 
+## Enable global session registry.
+##
+## Value: on | off
+broker.enable_session_registry = on
+
 ## Session locking strategy in a cluster.
 ##
 ## Value: Enum

etc/vm.args

@@ -0,0 +1,95 @@
+##############################
+# Erlang VM Args
+##############################
+
+## NOTE:
+##
+## Arguments configured in this file might be overridden by configs from `emqx.conf`.
+##
+## Some basic VM arguments are to be configured in `emqx.conf`,
+## such as `node.name` for `-name` and `node.cooke` for `-setcookie`.
+
+## Sets the maximum number of simultaneously existing processes for this system.
+#+P 2048000
+
+## Sets the maximum number of simultaneously existing ports for this system.
+#+Q 1024000
+
+## Sets the maximum number of ETS tables
+#+e 256000
+
+## Sets the maximum number of atoms the virtual machine can handle.
+#+t 1048576
+
+## Set the location of crash dumps
+#-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump
+
+## Set how many times generational garbages collections can be done without
+## forcing a fullsweep collection.
+#-env ERL_FULLSWEEP_AFTER 1000
+
+## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
+## (Disabled by default..use with caution!)
+#-heart
+
+## Specify the erlang distributed protocol.
+## Can be one of: inet_tcp, inet6_tcp, inet_tls
+#-proto_dist inet_tcp
+
+## Specify SSL Options in the file if using SSL for Erlang Distribution.
+## Used only when -proto_dist set to inet_tls
+#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf
+
+## Specifies the net_kernel tick time in seconds.
+## This is the approximate time a connected node may be unresponsive until
+## it is considered down and thereby disconnected.
+#-kernel net_ticktime 60
+
+## Sets the distribution buffer busy limit (dist_buf_busy_limit).
+#+zdbbl 8192
+
+## Sets default scheduler hint for port parallelism.
++spp true
+
+## Sets the number of threads in async thread pool. Valid range is 0-1024.
+#+A 8
+
+## Sets the default heap size of processes to the size Size.
+#+hms 233
+
+## Sets the default binary virtual heap size of processes to the size Size.
+#+hmbs 46422
+
+## Sets the number of IO pollsets to use when polling for I/O.
+#+IOp 1
+
+## Sets the number of IO poll threads to use when polling for I/O.
+#+IOt 1
+
+## Sets the number of scheduler threads to create and scheduler threads to set online.
+#+S 8:8
+
+## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online.
+#+SDcpu 8:8
+
+## Sets the number of dirty I/O scheduler threads to create.
+#+SDio 10
+
+## Suggested stack size, in kilowords, for scheduler threads.
+#+sss 32
+
+## Suggested stack size, in kilowords, for dirty CPU scheduler threads.
+#+sssdcpu 40
+
+## Suggested stack size, in kilowords, for dirty IO scheduler threads.
+#+sssdio 40
+
+## Sets scheduler bind type.
+## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db
+#+sbt db
+
+## Sets a user-defined CPU topology.
+#+sct L0-3c0-3p0N0:L4-7c0-3p1N1
+
+## Sets the mapping of warning messages for error_logger
+#+W w

etc/vm.args.edge

@@ -0,0 +1,95 @@
+##############################
+# Erlang VM Args
+##############################
+
+## NOTE:
+##
+## Arguments configured in this file might be overridden by configs from `emqx.conf`.
+##
+## Some basic VM arguments are to be configured in `emqx.conf`,
+## such as `node.name` for `-name` and `node.cooke` for `-setcookie`.
+
+## Sets the maximum number of simultaneously existing processes for this system.
++P 20480
+
+## Sets the maximum number of simultaneously existing ports for this system.
++Q 4096
+
+## Sets the maximum number of ETS tables
++e 512
+
+## Sets the maximum number of atoms the virtual machine can handle.
++t 65536
+
+## Set the location of crash dumps
+-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump
+
+## Set how many times generational garbages collections can be done without
+## forcing a fullsweep collection.
+-env ERL_FULLSWEEP_AFTER 0
+
+## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
+## (Disabled by default..use with caution!)
+#-heart
+
+## Specify the erlang distributed protocol.
+## Can be one of: inet_tcp, inet6_tcp, inet_tls
+#-proto_dist inet_tcp
+
+## Specify SSL Options in the file if using SSL for Erlang Distribution.
+## Used only when -proto_dist set to inet_tls
+#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf
+
+## Specifies the net_kernel tick time in seconds.
+## This is the approximate time a connected node may be unresponsive until
+## it is considered down and thereby disconnected.
+#-kernel net_ticktime 60
+
+## Sets the distribution buffer busy limit (dist_buf_busy_limit).
++zdbbl 1024
+
+## Sets default scheduler hint for port parallelism.
++spp false
+
+## Sets the number of threads in async thread pool. Valid range is 0-1024.
++A 1
+
+## Sets the default heap size of processes to the size Size.
+#+hms 233
+
+## Sets the default binary virtual heap size of processes to the size Size.
+#+hmbs 46422
+
+## Sets the number of IO pollsets to use when polling for I/O.
++IOp 1
+
+## Sets the number of IO poll threads to use when polling for I/O.
++IOt 1
+
+## Sets the number of scheduler threads to create and scheduler threads to set online.
++S 1:1
+
+## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online.
++SDcpu 1:1
+
+## Sets the number of dirty I/O scheduler threads to create.
++SDio 1
+
+## Suggested stack size, in kilowords, for scheduler threads.
+#+sss 32
+
+## Suggested stack size, in kilowords, for dirty CPU scheduler threads.
+#+sssdcpu 40
+
+## Suggested stack size, in kilowords, for dirty IO scheduler threads.
+#+sssdio 40
+
+## Sets scheduler bind type.
+## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db
+#+sbt db
+
+## Sets a user-defined CPU topology.
+#+sct L0-3c0-3p0N0:L4-7c0-3p1N1
+
+## Sets the mapping of warning messages for error_logger
+#+W w

priv/emqx.schema

@@ -191,13 +191,6 @@ end}.
   {default, "emqxsecretcookie"}
 ]}.
 
-%% @doc SMP Support
-{mapping, "node.smp", "vm_args.-smp", [
-  {default, auto},
-  {datatype, {enum, [enable, auto, disable]}},
-  hidden
-]}.
-
 %% @doc http://erlang.org/doc/man/heart.html
 {mapping, "node.heartbeat", "vm_args.-heart", [
   {datatype, flag},
@@ -211,13 +204,6 @@ end}.
     end
 end}.
 
-%% @doc Enable Kernel Poll
-{mapping, "node.kernel_poll", "vm_args.+K", [
-  {default, on},
-  {datatype, flag},
-  hidden
-]}.
-
 %% @doc More information at: http://erlang.org/doc/man/erl.html
 {mapping, "node.async_threads", "vm_args.+A", [
   {default, 64},
@@ -912,6 +898,11 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [
+  {default, 100},
+  {datatype, integer}
+]}.
+
 {mapping, "listener.tcp.$name.zone", "emqx.listeners", [
   {datatype, string}
 ]}.
@@ -1007,6 +998,11 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [
+  {default, 100},
+  {datatype, integer}
+]}.
+
 {mapping, "listener.ssl.$name.zone", "emqx.listeners", [
   {datatype, string}
 ]}.
@@ -1423,6 +1419,7 @@ end}.
                           {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)},
                           {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)},
                           {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)},
+                          {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)},
                           {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)},
                           {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))},
                           {rate_limit, Ratelimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))},
@@ -1735,6 +1732,11 @@ end}.
   {default, "1m"}
 ]}.
 
+{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [
+  {default, on},
+  {datatype, flag}
+]}.
+
 {mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [
   {default, quorum},
   {datatype, {enum, [local,one,quorum,all]}}

rebar.config

@@ -9,7 +9,7 @@
      {ekka, "v0.5.1"},
      {clique, "develop"},
      {esockd, "v5.4.2"},
-     {cuttlefish, "v2.1.1"}
+     {cuttlefish, "v2.2.0"}
     ]}.
 
 {edoc_opts, [{preprocess, true}]}.

src/emqx.erl

@@ -22,11 +22,10 @@
 %% PubSub API
 -export([subscribe/1, subscribe/2, subscribe/3]).
 -export([publish/1]).
--export([unsubscribe/1, unsubscribe/2]).
+-export([unsubscribe/1]).
 
 %% PubSub management API
 -export([topics/0, subscriptions/1, subscribers/1, subscribed/2]).
--export([get_subopts/2, set_subopts/3]).
 
 %% Hooks API
 -export([hook/2, hook/3, hook/4, unhook/2, run_hooks/2, run_hooks/3]).
@@ -70,20 +69,18 @@ is_running(Node) ->
 subscribe(Topic) ->
     emqx_broker:subscribe(iolist_to_binary(Topic)).
 
--spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok).
+-spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok).
 subscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId)->
     emqx_broker:subscribe(iolist_to_binary(Topic), SubId);
-subscribe(Topic, SubPid) when is_pid(SubPid) ->
+subscribe(Topic, SubOpts) when is_map(SubOpts) ->
-    emqx_broker:subscribe(iolist_to_binary(Topic), SubPid).
+    emqx_broker:subscribe(iolist_to_binary(Topic), SubOpts).
 
--spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid(),
+-spec(subscribe(emqx_topic:topic() | string(),
-                emqx_types:subopts()) -> ok).
+                emqx_types:subid() | pid(), emqx_types:subopts()) -> ok).
-subscribe(Topic, SubId, Options) when is_atom(SubId); is_binary(SubId)->
+subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), is_map(SubOpts) ->
-    emqx_broker:subscribe(iolist_to_binary(Topic), SubId, Options);
+    emqx_broker:subscribe(iolist_to_binary(Topic), SubId, SubOpts).
-subscribe(Topic, SubPid, Options) when is_pid(SubPid)->
-    emqx_broker:subscribe(iolist_to_binary(Topic), SubPid, Options).
 
--spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}).
+-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()).
 publish(Msg) ->
     emqx_broker:publish(Msg).
 
@@ -91,26 +88,10 @@ publish(Msg) ->
 unsubscribe(Topic) ->
     emqx_broker:unsubscribe(iolist_to_binary(Topic)).
 
--spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok).
-unsubscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId) ->
-    emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId);
-unsubscribe(Topic, SubPid) when is_pid(SubPid) ->
-    emqx_broker:unsubscribe(iolist_to_binary(Topic), SubPid).
-
 %%------------------------------------------------------------------------------
 %% PubSub management API
 %%------------------------------------------------------------------------------
 
--spec(get_subopts(emqx_topic:topic() | string(), emqx_types:subscriber())
-      -> emqx_types:subopts()).
-get_subopts(Topic, Subscriber) ->
-    emqx_broker:get_subopts(iolist_to_binary(Topic), Subscriber).
-
--spec(set_subopts(emqx_topic:topic() | string(), emqx_types:subscriber(),
-                  emqx_types:subopts()) -> boolean()).
-set_subopts(Topic, Subscriber, Options) when is_map(Options) ->
-    emqx_broker:set_subopts(iolist_to_binary(Topic), Subscriber, Options).
-
 -spec(topics() -> list(emqx_topic:topic())).
 topics() -> emqx_router:topics().
 
@@ -118,15 +99,15 @@ topics() -> emqx_router:topics().
 subscribers(Topic) ->
     emqx_broker:subscribers(iolist_to_binary(Topic)).
 
--spec(subscriptions(emqx_types:subscriber()) -> [{emqx_topic:topic(), emqx_types:subopts()}]).
+-spec(subscriptions(pid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]).
-subscriptions(Subscriber) ->
+subscriptions(SubPid) when is_pid(SubPid) ->
-    emqx_broker:subscriptions(Subscriber).
+    emqx_broker:subscriptions(SubPid).
 
--spec(subscribed(emqx_topic:topic() | string(), pid() | emqx_types:subid()) -> boolean()).
+-spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic() | string()) -> boolean()).
-subscribed(Topic, SubPid) when is_pid(SubPid) ->
+subscribed(SubPid, Topic) when is_pid(SubPid) ->
-    emqx_broker:subscribed(iolist_to_binary(Topic), SubPid);
+    emqx_broker:subscribed(SubPid, iolist_to_binary(Topic));
-subscribed(Topic, SubId) when is_atom(SubId); is_binary(SubId) ->
+subscribed(SubId, Topic) when is_atom(SubId); is_binary(SubId) ->
-    emqx_broker:subscribed(iolist_to_binary(Topic), SubId).
+    emqx_broker:subscribed(SubId, iolist_to_binary(Topic)).
 
 %%------------------------------------------------------------------------------
 %% Hooks API

src/emqx_access_control.erl

@@ -148,7 +148,7 @@ stop() ->
 %%-----------------------------------------------------------------------------
 
 init([]) ->
-    _ = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]),
+    ok = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]),
     {ok, #{}}.
 
 handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) ->

src/emqx_banned.erl

@@ -26,17 +26,16 @@
 
 -export([start_link/0]).
 -export([check/1]).
--export([add/1, del/1]).
+-export([add/1, delete/1]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
 -define(TAB, ?MODULE).
--define(SERVER, ?MODULE).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Mnesia bootstrap
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 mnesia(boot) ->
     ok = ekka_mnesia:create_table(?TAB, [
@@ -52,7 +51,7 @@ mnesia(copy) ->
 %% @doc Start the banned server.
 -spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
-    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
 -spec(check(emqx_types:credentials()) -> boolean()).
 check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) ->
@@ -64,25 +63,25 @@ check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) -
 add(Banned) when is_record(Banned, banned) ->
     mnesia:dirty_write(?TAB, Banned).
 
--spec(del({client_id, emqx_types:client_id()} |
+-spec(delete({client_id, emqx_types:client_id()}
-          {username, emqx_types:username()} |
+           | {username, emqx_types:username()}
-          {peername, emqx_types:peername()}) -> ok).
+           | {peername, emqx_types:peername()}) -> ok).
-del(Key) ->
+delete(Key) ->
     mnesia:dirty_delete(?TAB, Key).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% gen_server callbacks
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 init([]) ->
     {ok, ensure_expiry_timer(#{expiry_timer => undefined})}.
 
 handle_call(Req, _From, State) ->
-    emqx_logger:error("[BANNED] unexpected call: ~p", [Req]),
+    emqx_logger:error("[Banned] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    emqx_logger:error("[BANNED] unexpected msg: ~p", [Msg]),
+    emqx_logger:error("[Banned] unexpected msg: ~p", [Msg]),
     {noreply, State}.
 
 handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) ->
@@ -90,7 +89,7 @@ handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) ->
     {noreply, ensure_expiry_timer(State), hibernate};
 
 handle_info(Info, State) ->
-    emqx_logger:error("[BANNED] unexpected info: ~p", [Info]),
+    emqx_logger:error("[Banned] unexpected info: ~p", [Info]),
     {noreply, State}.
 
 terminate(_Reason, #{expiry_timer := TRef}) ->
@@ -99,21 +98,22 @@ terminate(_Reason, #{expiry_timer := TRef}) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Internal functions
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 -ifdef(TEST).
 ensure_expiry_timer(State) ->
     State#{expiry_timer := emqx_misc:start_timer(timer:seconds(1), expire)}.
 -else.
 ensure_expiry_timer(State) ->
-    State#{expiry_timer := emqx_misc:start_timer(timer:minutes(5), expire)}.
+    State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}.
 -endif.
 
 expire_banned_items(Now) ->
-    mnesia:foldl(fun
+    mnesia:foldl(
-            (B = #banned{until = Until}, _Acc) when Until < Now ->
+      fun(B = #banned{until = Until}, _Acc) when Until < Now ->
               mnesia:delete_object(?TAB, B, sticky_write);
          (_, _Acc) -> ok
       end, ok, ?TAB).
+

src/emqx_broker.erl

@@ -19,150 +19,163 @@
 -include("emqx.hrl").
 
 -export([start_link/2]).
--export([subscribe/1, subscribe/2, subscribe/3, subscribe/4]).
+-export([subscribe/1, subscribe/2, subscribe/3]).
--export([multi_subscribe/1, multi_subscribe/2, multi_subscribe/3]).
+-export([unsubscribe/1]).
+-export([subscriber_down/1]).
 -export([publish/1, safe_publish/1]).
--export([unsubscribe/1, unsubscribe/2, unsubscribe/3]).
+-export([dispatch/2]).
--export([multi_unsubscribe/1, multi_unsubscribe/2, multi_unsubscribe/3]).
--export([dispatch/2, dispatch/3]).
 -export([subscriptions/1, subscribers/1, subscribed/2]).
--export([get_subopts/2, set_subopts/3]).
+-export([get_subopts/2, set_subopts/2]).
 -export([topics/0]).
 
+%% Stats fun
+-export([stats_fun/0]).
+
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
+-import(emqx_tables, [lookup_value/2, lookup_value/3]).
+
 -ifdef(TEST).
 -compile(export_all).
 -compile(nowarn_export_all).
 -endif.
 
--record(state, {pool, id, submap, submon}).
--record(subscribe, {topic, subpid, subid, subopts = #{}}).
--record(unsubscribe, {topic, subpid, subid}).
-
-%% The default request timeout
--define(TIMEOUT, 60000).
 -define(BROKER, ?MODULE).
 
-%% ETS tables
+%% ETS tables for PubSub
 -define(SUBOPTION, emqx_suboption).
 -define(SUBSCRIBER, emqx_subscriber).
 -define(SUBSCRIPTION, emqx_subscription).
 
+%% Guards
 -define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))).
 
--spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}).
+-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()).
 start_link(Pool, Id) ->
-    gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE,
+    ok = create_tabs(),
-                          [Pool, Id], [{hibernate_after, 1000}]).
+    gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)},
+                          ?MODULE, [Pool, Id], []).
 
 %%------------------------------------------------------------------------------
-%% Subscribe
+%% Create tabs
+%%------------------------------------------------------------------------------
+
+-spec(create_tabs() -> ok).
+create_tabs() ->
+    TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}],
+
+    %% SubOption: {SubPid, Topic} -> SubOption
+    ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]),
+
+    %% Subscription: SubPid -> Topic1, Topic2, Topic3, ...
+    %% duplicate_bag: o(1) insert
+    ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]),
+
+    %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ...
+    %% bag: o(n) insert:(
+    ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]).
+
+%%------------------------------------------------------------------------------
+%% Subscribe API
 %%------------------------------------------------------------------------------
 
 -spec(subscribe(emqx_topic:topic()) -> ok).
 subscribe(Topic) when is_binary(Topic) ->
-    subscribe(Topic, self()).
+    subscribe(Topic, undefined).
 
--spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok).
+-spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok).
-subscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
-    subscribe(Topic, SubPid, undefined);
 subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
-    subscribe(Topic, self(), SubId).
+    subscribe(Topic, SubId, #{qos => 0});
+subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) ->
+    subscribe(Topic, undefined, SubOpts).
 
--spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid(),
+-spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok).
-                emqx_types:subid() | emqx_types:subopts()) -> ok).
-subscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
-    subscribe(Topic, SubPid, SubId, #{qos => 0});
-subscribe(Topic, SubPid, SubOpts) when is_binary(Topic), is_pid(SubPid), is_map(SubOpts) ->
-    subscribe(Topic, SubPid, undefined, SubOpts);
 subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts) ->
-    subscribe(Topic, self(), SubId, SubOpts).
+    SubPid = self(),
+    case ets:member(?SUBOPTION, {SubPid, Topic}) of
+        false ->
+            ok = emqx_broker_helper:register_sub(SubPid, SubId),
+            do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts));
+        true -> ok
+    end.
 
--spec(subscribe(emqx_topic:topic(), pid(), emqx_types:subid(), emqx_types:subopts()) -> ok).
+with_subid(undefined, SubOpts) ->
-subscribe(Topic, SubPid, SubId, SubOpts) when is_binary(Topic), is_pid(SubPid),
+    SubOpts;
-                                              ?is_subid(SubId), is_map(SubOpts) ->
+with_subid(SubId, SubOpts) ->
-    Broker = pick(SubPid),
+    maps:put(subid, SubId, SubOpts).
-    SubReq = #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts},
-    wait_for_reply(async_call(Broker, SubReq), ?TIMEOUT).
 
--spec(multi_subscribe(emqx_types:topic_table()) -> ok).
+%% @private
-multi_subscribe(TopicTable) when is_list(TopicTable) ->
+do_subscribe(Topic, SubPid, SubOpts) ->
-    multi_subscribe(TopicTable, self()).
+    true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}),
+    Group = maps:get(share, SubOpts, undefined),
+    do_subscribe(Group, Topic, SubPid, SubOpts).
 
--spec(multi_subscribe(emqx_types:topic_table(), pid() | emqx_types:subid()) -> ok).
+do_subscribe(undefined, Topic, SubPid, SubOpts) ->
-multi_subscribe(TopicTable, SubPid) when is_pid(SubPid) ->
+    case emqx_broker_helper:get_sub_shard(SubPid, Topic) of
-    multi_subscribe(TopicTable, SubPid, undefined);
+        0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}),
-multi_subscribe(TopicTable, SubId) when ?is_subid(SubId) ->
+             true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}),
-    multi_subscribe(TopicTable, self(), SubId).
+             call(pick(Topic), {subscribe, Topic});
+        I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
+             true = ets:insert(?SUBOPTION, {{SubPid, Topic}, maps:put(shard, I, SubOpts)}),
+             call(pick({Topic, I}), {subscribe, Topic, I})
+    end;
 
--spec(multi_subscribe(emqx_types:topic_table(), pid(), emqx_types:subid()) -> ok).
+%% Shared subscription
-multi_subscribe(TopicTable, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) ->
+do_subscribe(Group, Topic, SubPid, SubOpts) ->
-    Broker = pick(SubPid),
+    true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}),
-    SubReq = fun(Topic, SubOpts) ->
+    emqx_shared_sub:subscribe(Group, Topic, SubPid).
-                 #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}
-             end,
-    wait_for_replies([async_call(Broker, SubReq(Topic, SubOpts))
-                      || {Topic, SubOpts} <- TopicTable], ?TIMEOUT).
 
 %%------------------------------------------------------------------------------
-%% Unsubscribe
+%% Unsubscribe API
 %%------------------------------------------------------------------------------
 
 -spec(unsubscribe(emqx_topic:topic()) -> ok).
 unsubscribe(Topic) when is_binary(Topic) ->
-    unsubscribe(Topic, self()).
+    SubPid = self(),
+    case ets:lookup(?SUBOPTION, {SubPid, Topic}) of
+        [{_, SubOpts}] ->
+            _ = emqx_broker_helper:reclaim_seq(Topic),
+            do_unsubscribe(Topic, SubPid, SubOpts);
+        [] -> ok
+    end.
 
--spec(unsubscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok).
+do_unsubscribe(Topic, SubPid, SubOpts) ->
-unsubscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
+    true = ets:delete(?SUBOPTION, {SubPid, Topic}),
-    unsubscribe(Topic, SubPid, undefined);
+    true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}),
-unsubscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
+    Group = maps:get(share, SubOpts, undefined),
-    unsubscribe(Topic, self(), SubId).
+    do_unsubscribe(Group, Topic, SubPid, SubOpts).
 
--spec(unsubscribe(emqx_topic:topic(), pid(), emqx_types:subid()) -> ok).
+do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
-unsubscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
+    case maps:get(shard, SubOpts, 0) of
-    Broker = pick(SubPid),
+        0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
-    UnsubReq = #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId},
+             cast(pick(Topic), {unsubscribed, Topic});
-    wait_for_reply(async_call(Broker, UnsubReq), ?TIMEOUT).
+        I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
+             cast(pick({Topic, I}), {unsubscribed, Topic, I})
+    end;
 
--spec(multi_unsubscribe([emqx_topic:topic()]) -> ok).
+do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
-multi_unsubscribe(Topics) ->
+    emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
-    multi_unsubscribe(Topics, self()).
-
--spec(multi_unsubscribe([emqx_topic:topic()], pid() | emqx_types:subid()) -> ok).
-multi_unsubscribe(Topics, SubPid) when is_pid(SubPid) ->
-    multi_unsubscribe(Topics, SubPid, undefined);
-multi_unsubscribe(Topics, SubId) when ?is_subid(SubId) ->
-    multi_unsubscribe(Topics, self(), SubId).
-
--spec(multi_unsubscribe([emqx_topic:topic()], pid(), emqx_types:subid()) -> ok).
-multi_unsubscribe(Topics, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) ->
-    Broker = pick(SubPid),
-    UnsubReq = fun(Topic) ->
-                   #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}
-               end,
-    wait_for_replies([async_call(Broker, UnsubReq(Topic)) || Topic <- Topics], ?TIMEOUT).
 
 %%------------------------------------------------------------------------------
 %% Publish
 %%------------------------------------------------------------------------------
 
--spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}).
+-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()).
 publish(Msg) when is_record(Msg, message) ->
     _ = emqx_tracer:trace(publish, Msg),
-    {ok, case emqx_hooks:run('message.publish', [], Msg) of
+    case emqx_hooks:run('message.publish', [], Msg) of
         {ok, Msg1 = #message{topic = Topic}} ->
             Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)),
             Delivery#delivery.results;
         {stop, _} ->
             emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]),
             []
-         end}.
+    end.
 
--spec(safe_publish(emqx_types:message()) -> ok).
 %% Called internally
+-spec(safe_publish(emqx_types:message()) -> ok).
 safe_publish(Msg) when is_record(Msg, message) ->
     try
         publish(Msg)
@@ -228,97 +241,137 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) ->
             inc_dropped_cnt(Topic),
             Delivery;
         [Sub] -> %% optimize?
-            dispatch(Sub, Topic, Msg),
+            Cnt = dispatch(Sub, Topic, Msg),
-            Delivery#delivery{results = [{dispatch, Topic, 1}|Results]};
+            Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]};
-        Subscribers ->
+        Subs ->
-            Count = lists:foldl(fun(Sub, Acc) ->
+            Cnt = lists:foldl(
-                                    dispatch(Sub, Topic, Msg), Acc + 1
+                    fun(Sub, Acc) ->
-                                end, 0, Subscribers),
+                            dispatch(Sub, Topic, Msg) + Acc
-            Delivery#delivery{results = [{dispatch, Topic, Count}|Results]}
+                    end, 0, Subs),
+            Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]}
     end.
 
-dispatch({SubPid, _SubId}, Topic, Msg) when is_pid(SubPid) ->
+dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
-    SubPid ! {dispatch, Topic, Msg};
+    case erlang:is_process_alive(SubPid) of
-dispatch({share, _Group, _Sub}, _Topic, _Msg) ->
+        true ->
-    ignored.
+            SubPid ! {dispatch, Topic, Msg},
+            1;
+        false -> 0
+    end;
+dispatch({shard, I}, Topic, Msg) ->
+    lists:foldl(
+      fun(SubPid, Cnt) ->
+              dispatch(SubPid, Topic, Msg) + Cnt
+      end, 0, subscribers({shard, Topic, I})).
 
 inc_dropped_cnt(<<"$SYS/", _/binary>>) ->
     ok;
 inc_dropped_cnt(_Topic) ->
     emqx_metrics:inc('messages/dropped').
 
--spec(subscribers(emqx_topic:topic()) -> [emqx_types:subscriber()]).
+-spec(subscribers(emqx_topic:topic()) -> [pid()]).
-subscribers(Topic) ->
+subscribers(Topic) when is_binary(Topic) ->
-    try ets:lookup_element(?SUBSCRIBER, Topic, 2) catch error:badarg -> [] end.
+    lookup_value(?SUBSCRIBER, Topic, []);
+subscribers(Shard = {shard, _Topic, _I})  ->
+    lookup_value(?SUBSCRIBER, Shard, []).
 
--spec(subscriptions(emqx_types:subscriber())
+%%------------------------------------------------------------------------------
+%% Subscriber is down
+%%------------------------------------------------------------------------------
+
+-spec(subscriber_down(pid()) -> true).
+subscriber_down(SubPid) ->
+    lists:foreach(
+      fun(Topic) ->
+          case lookup_value(?SUBOPTION, {SubPid, Topic}) of
+              SubOpts when is_map(SubOpts) ->
+                  _ = emqx_broker_helper:reclaim_seq(Topic),
+                  true = ets:delete(?SUBOPTION, {SubPid, Topic}),
+                  case maps:get(shard, SubOpts, 0) of
+                      0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
+                           ok = cast(pick(Topic), {unsubscribed, Topic});
+                      I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
+                           ok = cast(pick({Topic, I}), {unsubscribed, Topic, I})
+                  end;
+              undefined -> ok
+          end
+      end, lookup_value(?SUBSCRIPTION, SubPid, [])),
+    ets:delete(?SUBSCRIPTION, SubPid).
+
+%%------------------------------------------------------------------------------
+%% Management APIs
+%%------------------------------------------------------------------------------
+
+-spec(subscriptions(pid() | emqx_types:subid())
       -> [{emqx_topic:topic(), emqx_types:subopts()}]).
-subscriptions(Subscriber) ->
+subscriptions(SubPid) when is_pid(SubPid) ->
-    lists:map(fun({_, {share, _Group, Topic}}) ->
+    [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})}
-                  subscription(Topic, Subscriber);
+      || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])];
-                 ({_, Topic}) ->
+subscriptions(SubId) ->
-                  subscription(Topic, Subscriber)
+    case emqx_broker_helper:lookup_subpid(SubId) of
-              end, ets:lookup(?SUBSCRIPTION, Subscriber)).
+        SubPid when is_pid(SubPid) ->
-
+            subscriptions(SubPid);
-subscription(Topic, Subscriber) ->
+        undefined -> []
-    {Topic, ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)}.
-
--spec(subscribed(emqx_topic:topic(), pid() | emqx_types:subid() | emqx_types:subscriber()) -> boolean()).
-subscribed(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
-    case ets:match_object(?SUBOPTION, {{Topic, {SubPid, '_'}}, '_'}, 1) of
-        {Match, _} ->
-            length(Match) >= 1;
-        '$end_of_table' ->
-            false
-    end;
-subscribed(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
-    case ets:match_object(?SUBOPTION, {{Topic, {'_', SubId}}, '_'}, 1) of
-        {Match, _} ->
-            length(Match) >= 1;
-    '$end_of_table' ->
-            false
-    end;
-subscribed(Topic, {SubPid, SubId}) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
-    ets:member(?SUBOPTION, {Topic, {SubPid, SubId}}).
-
--spec(get_subopts(emqx_topic:topic(), emqx_types:subscriber()) -> emqx_types:subopts()).
-get_subopts(Topic, Subscriber) when is_binary(Topic) ->
-    try ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)
-    catch error:badarg -> []
     end.
 
--spec(set_subopts(emqx_topic:topic(), emqx_types:subscriber(), emqx_types:subopts()) -> boolean()).
+-spec(subscribed(pid(), emqx_topic:topic()) -> boolean()).
-set_subopts(Topic, Subscriber, Opts) when is_binary(Topic), is_map(Opts) ->
+subscribed(SubPid, Topic) when is_pid(SubPid) ->
-    case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
+    ets:member(?SUBOPTION, {SubPid, Topic});
+subscribed(SubId, Topic) when ?is_subid(SubId) ->
+    SubPid = emqx_broker_helper:lookup_subpid(SubId),
+    ets:member(?SUBOPTION, {SubPid, Topic}).
+
+-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts() | undefined).
+get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) ->
+    lookup_value(?SUBOPTION, {SubPid, Topic});
+get_subopts(SubId, Topic) when ?is_subid(SubId) ->
+    case emqx_broker_helper:lookup_subpid(SubId) of
+        SubPid when is_pid(SubPid) ->
+            get_subopts(SubPid, Topic);
+        undefined -> undefined
+    end.
+
+-spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()).
+set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) ->
+    Sub = {self(), Topic},
+    case ets:lookup(?SUBOPTION, Sub) of
         [{_, OldOpts}] ->
-            ets:insert(?SUBOPTION, {{Topic, Subscriber}, maps:merge(OldOpts, Opts)});
+            ets:insert(?SUBOPTION, {Sub, maps:merge(OldOpts, NewOpts)});
         [] -> false
     end.
 
-async_call(Broker, Req) ->
+-spec(topics() -> [emqx_topic:topic()]).
-    From = {self(), Tag = make_ref()},
+topics() ->
-    ok = gen_server:cast(Broker, {From, Req}),
+    emqx_router:topics().
-    Tag.
 
-wait_for_replies(Tags, Timeout) ->
+%%------------------------------------------------------------------------------
-    lists:foreach(
+%% Stats fun
-      fun(Tag) ->
+%%------------------------------------------------------------------------------
-          wait_for_reply(Tag, Timeout)
-      end, Tags).
 
-wait_for_reply(Tag, Timeout) ->
+stats_fun() ->
-    receive
+    safe_update_stats(?SUBSCRIBER, 'subscribers/count', 'subscribers/max'),
-        {Tag, Reply} -> Reply
+    safe_update_stats(?SUBSCRIPTION, 'subscriptions/count', 'subscriptions/max'),
-    after Timeout ->
+    safe_update_stats(?SUBOPTION, 'suboptions/count', 'suboptions/max').
-        exit(timeout)
+
+safe_update_stats(Tab, Stat, MaxStat) ->
+    case ets:info(Tab, size) of
+        undefined -> ok;
+        Size -> emqx_stats:setstat(Stat, MaxStat, Size)
     end.
 
-%% Pick a broker
+%%------------------------------------------------------------------------------
-pick(SubPid) when is_pid(SubPid) ->
+%% call, cast, pick
-    gproc_pool:pick_worker(broker, SubPid).
+%%------------------------------------------------------------------------------
 
--spec(topics() -> [emqx_topic:topic()]).
+call(Broker, Req) ->
-topics() -> emqx_router:topics().
+    gen_server:call(Broker, Req).
+
+cast(Broker, Msg) ->
+    gen_server:cast(Broker, Msg).
+
+%% Pick a broker
+pick(Topic) ->
+    gproc_pool:pick_worker(broker_pool, Topic).
 
 %%------------------------------------------------------------------------------
 %% gen_server callbacks
@@ -326,38 +379,49 @@ topics() -> emqx_router:topics().
 
 init([Pool, Id]) ->
     true = gproc_pool:connect_worker(Pool, {Pool, Id}),
-    {ok, #state{pool = Pool, id = Id, submap = #{}, submon = emqx_pmon:new()}}.
+    {ok, #{pool => Pool, id => Id}}.
+
+handle_call({subscribe, Topic}, _From, State) ->
+    Ok = emqx_router:do_add_route(Topic),
+    {reply, Ok, State};
+
+handle_call({subscribe, Topic, I}, _From, State) ->
+    Ok = case get(Shard = {Topic, I}) of
+             undefined ->
+                 _ = put(Shard, true),
+                 true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}),
+                 cast(pick(Topic), {subscribe, Topic});
+             true -> ok
+         end,
+    {reply, Ok, State};
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[Broker] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({From, #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}}, State) ->
+handle_cast({subscribe, Topic}, State) ->
-    Subscriber = {SubPid, SubId},
+    case emqx_router:do_add_route(Topic) of
-    case ets:member(?SUBOPTION, {Topic, Subscriber}) of
+        ok -> ok;
-        false ->
+        {error, Reason} ->
-            Group = maps:get(share, SubOpts, undefined),
+            emqx_logger:error("[Broker] Failed to add route: ~p", [Reason])
-            true = do_subscribe(Group, Topic, Subscriber, SubOpts),
+    end,
-            emqx_shared_sub:subscribe(Group, Topic, SubPid),
+    {noreply, State};
-            emqx_router:add_route(From, Topic, dest(Group)),
-            {noreply, monitor_subscriber(Subscriber, State)};
-        true ->
-            gen_server:reply(From, ok),
-            {noreply, State}
-    end;
 
-handle_cast({From, #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}}, State) ->
+handle_cast({unsubscribed, Topic}, State) ->
-    Subscriber = {SubPid, SubId},
-    case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
-        [{_, SubOpts}] ->
-            Group = maps:get(share, SubOpts, undefined),
-            true = do_unsubscribe(Group, Topic, Subscriber),
-            emqx_shared_sub:unsubscribe(Group, Topic, SubPid),
     case ets:member(?SUBSCRIBER, Topic) of
-                false -> emqx_router:del_route(From, Topic, dest(Group));
+        false ->
-                true  -> gen_server:reply(From, ok)
+            _ = emqx_router:do_delete_route(Topic);
-            end;
+        true -> ok
-        [] -> gen_server:reply(From, ok)
+    end,
+    {noreply, State};
+
+handle_cast({unsubscribed, Topic, I}, State) ->
+    case ets:member(?SUBSCRIBER, {shard, Topic, I}) of
+        false ->
+            _ = erase({Topic, I}),
+            true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}),
+            cast(pick(Topic), {unsubscribed, Topic});
+        true -> ok
     end,
     {noreply, State};
 
@@ -365,21 +429,11 @@ handle_cast(Msg, State) ->
     emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #state{submap = SubMap}) ->
-    case maps:find(SubPid, SubMap) of
-        {ok, SubIds} ->
-            lists:foreach(fun(SubId) -> subscriber_down({SubPid, SubId}) end, SubIds),
-            {noreply, demonitor_subscriber(SubPid, State)};
-        error ->
-            emqx_logger:error("unexpected 'DOWN': ~p, reason: ~p", [SubPid, Reason]),
-            {noreply, State}
-    end;
-
 handle_info(Info, State) ->
     emqx_logger:error("[Broker] unexpected info: ~p", [Info]),
     {noreply, State}.
 
-terminate(_Reason, #state{pool = Pool, id = Id}) ->
+terminate(_Reason, #{pool := Pool, id := Id}) ->
     gproc_pool:disconnect_worker(Pool, {Pool, Id}).
 
 code_change(_OldVsn, State, _Extra) ->
@@ -389,52 +443,3 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-do_subscribe(Group, Topic, Subscriber, SubOpts) ->
-    ets:insert(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}),
-    ets:insert(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}),
-    ets:insert(?SUBOPTION, {{Topic, Subscriber}, SubOpts}).
-
-do_unsubscribe(Group, Topic, Subscriber) ->
-    ets:delete_object(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}),
-    ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}),
-    ets:delete(?SUBOPTION, {Topic, Subscriber}).
-
-subscriber_down(Subscriber) ->
-    Topics = lists:map(fun({_, {share, Group, Topic}}) ->
-                           {Topic, Group};
-                          ({_, Topic}) ->
-                           {Topic, undefined}
-                       end, ets:lookup(?SUBSCRIPTION, Subscriber)),
-    lists:foreach(fun({Topic, undefined}) ->
-                      true = do_unsubscribe(undefined, Topic, Subscriber),
-                      ets:member(?SUBSCRIBER, Topic) orelse emqx_router:del_route(Topic, dest(undefined));
-                 ({Topic, Group}) ->
-                     true = do_unsubscribe(Group, Topic, Subscriber),
-                     Groups = groups(Topic),
-                     case lists:member(Group, lists:usort(Groups)) of
-                        true  -> ok;
-                        false -> emqx_router:del_route(Topic, dest(Group))
-                    end
-                  end, Topics).
-
-monitor_subscriber({SubPid, SubId}, State = #state{submap = SubMap, submon = SubMon}) ->
-    UpFun = fun(SubIds) -> lists:usort([SubId|SubIds]) end,
-    State#state{submap = maps:update_with(SubPid, UpFun, [SubId], SubMap),
-                submon = emqx_pmon:monitor(SubPid, SubMon)}.
-
-demonitor_subscriber(SubPid, State = #state{submap = SubMap, submon = SubMon}) ->
-    State#state{submap = maps:remove(SubPid, SubMap),
-                submon = emqx_pmon:demonitor(SubPid, SubMon)}.
-
-dest(undefined) -> node();
-dest(Group)     -> {Group, node()}.
-
-shared(undefined, Name) -> Name;
-shared(Group, Name)     -> {share, Group, Name}.
-
-groups(Topic) ->
-    lists:foldl(fun({_, {share, Group, _}}, Acc) ->
-                        [Group | Acc];
-                   ({_, _}, Acc) ->
-                        Acc
-                end, [], ets:lookup(?SUBSCRIBER, Topic)).

src/emqx_broker_helper.erl

@@ -17,42 +17,110 @@
 -behaviour(gen_server).
 
 -export([start_link/0]).
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
+-export([register_sub/2]).
+-export([lookup_subid/1, lookup_subpid/1]).
+-export([get_sub_shard/2]).
+-export([create_seq/1, reclaim_seq/1]).
 
-%% internal export
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
--export([stats_fun/0]).
+         code_change/3]).
 
 -define(HELPER, ?MODULE).
+-define(SUBID, emqx_subid).
+-define(SUBMON, emqx_submon).
+-define(SUBSEQ, emqx_subseq).
+-define(SHARD, 1024).
 
--record(state, {}).
+-define(BATCH_SIZE, 100000).
 
--spec(start_link() -> {ok, pid()} | ignore | {error, any()}).
+-spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
     gen_server:start_link({local, ?HELPER}, ?MODULE, [], []).
 
+-spec(register_sub(pid(), emqx_types:subid()) -> ok).
+register_sub(SubPid, SubId) when is_pid(SubPid) ->
+    case ets:lookup(?SUBMON, SubPid) of
+        [] ->
+            gen_server:cast(?HELPER, {register_sub, SubPid, SubId});
+        [{_, SubId}] ->
+            ok;
+        _Other ->
+            error(subid_conflict)
+    end.
+
+-spec(lookup_subid(pid()) -> emqx_types:subid() | undefined).
+lookup_subid(SubPid) when is_pid(SubPid) ->
+    emqx_tables:lookup_value(?SUBMON, SubPid).
+
+-spec(lookup_subpid(emqx_types:subid()) -> pid()).
+lookup_subpid(SubId) ->
+    emqx_tables:lookup_value(?SUBID, SubId).
+
+-spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()).
+get_sub_shard(SubPid, Topic) ->
+    case create_seq(Topic) of
+        Seq when Seq =< ?SHARD -> 0;
+        _ -> erlang:phash2(SubPid, shards_num()) + 1
+    end.
+
+-spec(shards_num() -> pos_integer()).
+shards_num() ->
+    %% Dynamic sharding later...
+    ets:lookup_element(?HELPER, shards, 2).
+
+-spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()).
+create_seq(Topic) ->
+    emqx_sequence:nextval(?SUBSEQ, Topic).
+
+-spec(reclaim_seq(emqx_topic:topic()) -> emqx_sequence:seqid()).
+reclaim_seq(Topic) ->
+    emqx_sequence:reclaim(?SUBSEQ, Topic).
+
 %%------------------------------------------------------------------------------
 %% gen_server callbacks
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    %% Use M:F/A for callback, not anonymous function because
+    %% Helper table
-    %% fun M:F/A is small, also no badfun risk during hot beam reload
+    ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]),
-    emqx_stats:update_interval(broker_stats, fun ?MODULE:stats_fun/0),
+    %% Shards: CPU * 32
-    {ok, #state{}, hibernate}.
+    true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}),
+    %% SubSeq: Topic -> SeqId
+    ok = emqx_sequence:create(?SUBSEQ),
+    %% SubId: SubId -> SubPid
+    ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]),
+    %% SubMon: SubPid -> SubId
+    ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]),
+    %% Stats timer
+    ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0),
+    {ok, #{pmon => emqx_pmon:new()}}.
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
+handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) ->
+    true = (SubId =:= undefined) orelse ets:insert(?SUBID, {SubId, SubPid}),
+    true = ets:insert(?SUBMON, {SubPid, SubId}),
+    {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}};
+
 handle_cast(Msg, State) ->
     emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
+handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) ->
+    SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)],
+    ok = emqx_pool:async_submit(
+           fun lists:foreach/2, [fun clean_down/1, SubPids]),
+    {_, PMon1} = emqx_pmon:erase_all(SubPids, PMon),
+    {noreply, State#{pmon := PMon1}};
+
 handle_info(Info, State) ->
     emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]),
     {noreply, State}.
 
-terminate(_Reason, #state{}) ->
+terminate(_Reason, _State) ->
+    true = emqx_sequence:delete(?SUBSEQ),
     emqx_stats:cancel_update(broker_stats).
 
 code_change(_OldVsn, State, _Extra) ->
@@ -62,17 +130,13 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-stats_fun() ->
+clean_down(SubPid) ->
-    safe_update_stats(emqx_subscriber,
+    case ets:lookup(?SUBMON, SubPid) of
-                      'subscribers/count', 'subscribers/max'),
+        [{_, SubId}] ->
-    safe_update_stats(emqx_subscription,
+            true = ets:delete(?SUBMON, SubPid),
-                      'subscriptions/count', 'subscriptions/max'),
+            true = (SubId =:= undefined)
-    safe_update_stats(emqx_suboptions,
+                orelse ets:delete_object(?SUBID, {SubId, SubPid}),
-                      'suboptions/count', 'suboptions/max').
+            emqx_broker:subscriber_down(SubPid);
-
+        [] -> ok
-safe_update_stats(Tab, Stat, MaxStat) ->
-    case ets:info(Tab, size) of
-        undefined -> ok;
-        Size -> emqx_stats:setstat(Stat, MaxStat, Size)
     end.
 

src/emqx_broker_sup.erl

@@ -20,8 +20,6 @@
 
 -export([init/1]).
 
--define(TAB_OPTS, [public, {read_concurrency, true}, {write_concurrency, true}]).
-
 start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
@@ -30,39 +28,26 @@ start_link() ->
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    %% Create the pubsub tables
-    ok = lists:foreach(fun create_tab/1, [subscription, subscriber, suboption]),
-
-    %% Shared subscription
-    SharedSub = {shared_sub, {emqx_shared_sub, start_link, []},
-                 permanent, 5000, worker, [emqx_shared_sub]},
-
-    %% Broker helper
-    Helper = {broker_helper, {emqx_broker_helper, start_link, []},
-              permanent, 5000, worker, [emqx_broker_helper]},
-
     %% Broker pool
-    BrokerPool = emqx_pool_sup:spec(emqx_broker_pool,
+    PoolSize = emqx_vm:schedulers() * 2,
-                                    [broker, hash, emqx_vm:schedulers() * 2,
+    BrokerPool = emqx_pool_sup:spec([broker_pool, hash, PoolSize,
                                      {emqx_broker, start_link, []}]),
 
-    {ok, {{one_for_all, 0, 1}, [SharedSub, Helper, BrokerPool]}}.
+    %% Shared subscription
+    SharedSub = #{id       => shared_sub,
+                  start    => {emqx_shared_sub, start_link, []},
+                  restart  => permanent,
+                  shutdown => 2000,
+                  type     => worker,
+                  modules  => [emqx_shared_sub]},
 
-%%------------------------------------------------------------------------------
+    %% Broker helper
-%% Create tables
+    Helper = #{id       => helper,
-%%------------------------------------------------------------------------------
+               start    => {emqx_broker_helper, start_link, []},
+               restart  => permanent,
+               shutdown => 2000,
+               type     => worker,
+               modules  => [emqx_broker_helper]},
 
-create_tab(suboption) ->
+    {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}.
-    %% Suboption: {Topic, Sub} -> [{qos, 1}]
-    emqx_tables:new(emqx_suboption, [set | ?TAB_OPTS]);
-
-create_tab(subscriber) ->
-    %% Subscriber: Topic -> Sub1, Sub2, Sub3, ..., SubN
-    %% duplicate_bag: o(1) insert
-    emqx_tables:new(emqx_subscriber, [duplicate_bag | ?TAB_OPTS]);
-
-create_tab(subscription) ->
-    %% Subscription: Sub -> Topic1, Topic2, Topic3, ..., TopicN
-    %% bag: o(n) insert
-    emqx_tables:new(emqx_subscription, [bag | ?TAB_OPTS]).
 

src/emqx_cli.erl

@@ -17,15 +17,16 @@
 -export([print/1, print/2, usage/1, usage/2]).
 
 print(Msg) ->
-    io:format(Msg).
+    io:format(Msg), lists:flatten(io_lib:format("~p", [Msg])).
 
 print(Format, Args) ->
-    io:format(Format, Args).
+    io:format(Format, Args), lists:flatten(io_lib:format(Format, Args)).
 
 usage(CmdList) ->
-    lists:foreach(
+    lists:map(
       fun({Cmd, Descr}) ->
-        io:format("~-48s# ~s~n", [Cmd, Descr])
+          io:format("~-48s# ~s~n", [Cmd, Descr]),
+          lists:flatten(io_lib:format("~-48s# ~s~n", [Cmd, Descr]))
       end, CmdList).
 
 usage(Format, Args) ->

src/emqx_cm.erl

@@ -20,102 +20,110 @@
 
 -export([start_link/0]).
 
--export([lookup_connection/1]).
 -export([register_connection/1, register_connection/2]).
--export([unregister_connection/1]).
+-export([unregister_connection/1, unregister_connection/2]).
--export([get_conn_attrs/1, set_conn_attrs/2]).
+-export([get_conn_attrs/1, get_conn_attrs/2]).
--export([get_conn_stats/1, set_conn_stats/2]).
+-export([set_conn_attrs/2, set_conn_attrs/3]).
+-export([get_conn_stats/1, get_conn_stats/2]).
+-export([set_conn_stats/2, set_conn_stats/3]).
 -export([lookup_conn_pid/1]).
 
+%% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
 %% internal export
--export([update_conn_stats/0]).
+-export([stats_fun/0]).
 
 -define(CM, ?MODULE).
 
-%% ETS Tables.
+%% ETS tables for connection management.
 -define(CONN_TAB, emqx_conn).
 -define(CONN_ATTRS_TAB, emqx_conn_attrs).
 -define(CONN_STATS_TAB, emqx_conn_stats).
 
+-define(BATCH_SIZE, 100000).
+
 %% @doc Start the connection manager.
 -spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
     gen_server:start_link({local, ?CM}, ?MODULE, [], []).
 
-%% @doc Lookup a connection.
+%%------------------------------------------------------------------------------
--spec(lookup_connection(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})).
+%% API
-lookup_connection(ClientId) when is_binary(ClientId) ->
+%%------------------------------------------------------------------------------
-    ets:lookup(?CONN_TAB, ClientId).
 
 %% @doc Register a connection.
--spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok).
+-spec(register_connection(emqx_types:client_id()) -> ok).
 register_connection(ClientId) when is_binary(ClientId) ->
-    register_connection({ClientId, self()});
+    register_connection(ClientId, self()).
 
-register_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) ->
+-spec(register_connection(emqx_types:client_id(), pid()) -> ok).
-    _ = ets:insert(?CONN_TAB, Conn),
+register_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) ->
+    true = ets:insert(?CONN_TAB, {ClientId, ConnPid}),
     notify({registered, ClientId, ConnPid}).
 
--spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}, list()) -> ok).
+%% @doc Unregister a connection.
-register_connection(ClientId, Attrs) when is_binary(ClientId) ->
+-spec(unregister_connection(emqx_types:client_id()) -> ok).
-    register_connection({ClientId, self()}, Attrs);
+unregister_connection(ClientId) when is_binary(ClientId) ->
-register_connection(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) ->
+    unregister_connection(ClientId, self()).
-    set_conn_attrs(Conn, Attrs),
+
-    register_connection(Conn).
+-spec(unregister_connection(emqx_types:client_id(), pid()) -> ok).
+unregister_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) ->
+    true = do_unregister_connection({ClientId, ConnPid}),
+    notify({unregistered, ConnPid}).
+
+do_unregister_connection(Conn) ->
+    true = ets:delete(?CONN_STATS_TAB, Conn),
+    true = ets:delete(?CONN_ATTRS_TAB, Conn),
+    true = ets:delete_object(?CONN_TAB, Conn).
 
 %% @doc Get conn attrs
--spec(get_conn_attrs({emqx_types:client_id(), pid()}) -> list()).
+-spec(get_conn_attrs(emqx_types:client_id()) -> list()).
-get_conn_attrs(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) ->
+get_conn_attrs(ClientId) when is_binary(ClientId) ->
-    try
+    ConnPid = lookup_conn_pid(ClientId),
-        ets:lookup_element(?CONN_ATTRS_TAB, Conn, 2)
+    get_conn_attrs(ClientId, ConnPid).
-    catch
+
-        error:badarg -> []
+-spec(get_conn_attrs(emqx_types:client_id(), pid()) -> list()).
-    end.
+get_conn_attrs(ClientId, ConnPid) when is_binary(ClientId) ->
+    emqx_tables:lookup_value(?CONN_ATTRS_TAB, {ClientId, ConnPid}, []).
 
 %% @doc Set conn attrs
+-spec(set_conn_attrs(emqx_types:client_id(), list()) -> true).
 set_conn_attrs(ClientId, Attrs) when is_binary(ClientId) ->
-    set_conn_attrs({ClientId, self()}, Attrs);
+    set_conn_attrs(ClientId, self(), Attrs).
-set_conn_attrs(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) ->
+
+-spec(set_conn_attrs(emqx_types:client_id(), pid(), list()) -> true).
+set_conn_attrs(ClientId, ConnPid, Attrs) when is_binary(ClientId), is_pid(ConnPid) ->
+    Conn = {ClientId, ConnPid},
     ets:insert(?CONN_ATTRS_TAB, {Conn, Attrs}).
 
-%% @doc Unregister a conn.
--spec(unregister_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok).
-unregister_connection(ClientId) when is_binary(ClientId) ->
-    unregister_connection({ClientId, self()});
-
-unregister_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) ->
-    _ = ets:delete(?CONN_STATS_TAB, Conn),
-    _ = ets:delete(?CONN_ATTRS_TAB, Conn),
-    _ = ets:delete_object(?CONN_TAB, Conn),
-    notify({unregistered, ClientId, ConnPid}).
-
-%% @doc Lookup connection pid
--spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined).
-lookup_conn_pid(ClientId) when is_binary(ClientId) ->
-    case ets:lookup(?CONN_TAB, ClientId) of
-        [] -> undefined;
-        [{_, Pid}] -> Pid
-    end.
-
 %% @doc Get conn stats
--spec(get_conn_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())).
+-spec(get_conn_stats(emqx_types:client_id()) -> list(emqx_stats:stats())).
-get_conn_stats(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) ->
+get_conn_stats(ClientId) when is_binary(ClientId) ->
-    try ets:lookup_element(?CONN_STATS_TAB, Conn, 2)
+    ConnPid = lookup_conn_pid(ClientId),
-    catch
+    get_conn_stats(ClientId, ConnPid).
-        error:badarg -> []
+
-    end.
+-spec(get_conn_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())).
+get_conn_stats(ClientId, ConnPid) when is_binary(ClientId) ->
+    Conn = {ClientId, ConnPid},
+    emqx_tables:lookup_value(?CONN_STATS_TAB, Conn, []).
 
 %% @doc Set conn stats.
--spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> boolean()).
+-spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> true).
 set_conn_stats(ClientId, Stats) when is_binary(ClientId) ->
-    set_conn_stats({ClientId, self()}, Stats);
+    set_conn_stats(ClientId, self(), Stats).
 
-set_conn_stats(Conn = {ClientId, ConnPid}, Stats) when is_binary(ClientId), is_pid(ConnPid) ->
+-spec(set_conn_stats(emqx_types:client_id(), pid(), list(emqx_stats:stats())) -> true).
+set_conn_stats(ClientId, ConnPid, Stats) when is_binary(ClientId), is_pid(ConnPid) ->
+    Conn = {ClientId, ConnPid},
     ets:insert(?CONN_STATS_TAB, {Conn, Stats}).
 
+%% @doc Lookup connection pid.
+-spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined).
+lookup_conn_pid(ClientId) when is_binary(ClientId) ->
+    emqx_tables:lookup_value(?CONN_TAB, ClientId).
+
 notify(Msg) ->
     gen_server:cast(?CM, {notify, Msg}).
 
@@ -125,10 +133,10 @@ notify(Msg) ->
 
 init([]) ->
     TabOpts = [public, set, {write_concurrency, true}],
-    _ = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]),
+    ok = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]),
-    _ = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts),
+    ok = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts),
-    _ = emqx_tables:new(?CONN_STATS_TAB, TabOpts),
+    ok = emqx_tables:new(?CONN_STATS_TAB, TabOpts),
-    ok = emqx_stats:update_interval(cm_stats, fun ?MODULE:update_conn_stats/0),
+    ok = emqx_stats:update_interval(conn_stats, fun ?MODULE:stats_fun/0),
     {ok, #{conn_pmon => emqx_pmon:new()}}.
 
 handle_call(Req, _From, State) ->
@@ -138,28 +146,26 @@ handle_call(Req, _From, State) ->
 handle_cast({notify, {registered, ClientId, ConnPid}}, State = #{conn_pmon := PMon}) ->
     {noreply, State#{conn_pmon := emqx_pmon:monitor(ConnPid, ClientId, PMon)}};
 
-handle_cast({notify, {unregistered, _ClientId, ConnPid}}, State = #{conn_pmon := PMon}) ->
+handle_cast({notify, {unregistered, ConnPid}}, State = #{conn_pmon := PMon}) ->
     {noreply, State#{conn_pmon := emqx_pmon:demonitor(ConnPid, PMon)}};
 
 handle_cast(Msg, State) ->
     emqx_logger:error("[CM] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({'DOWN', _MRef, process, ConnPid, _Reason}, State = #{conn_pmon := PMon}) ->
+handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{conn_pmon := PMon}) ->
-    case emqx_pmon:find(ConnPid, PMon) of
+    ConnPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)],
-        undefined ->
+    {Items, PMon1} = emqx_pmon:erase_all(ConnPids, PMon),
-            {noreply, State};
+    ok = emqx_pool:async_submit(
-        ClientId ->
+           fun lists:foreach/2, [fun clean_down/1, Items]),
-            unregister_connection({ClientId, ConnPid}),
+    {noreply, State#{conn_pmon := PMon1}};
-            {noreply, State#{conn_pmon := emqx_pmon:erase(ConnPid, PMon)}}
-    end;
 
 handle_info(Info, State) ->
     emqx_logger:error("[CM] unexpected info: ~p", [Info]),
     {noreply, State}.
 
 terminate(_Reason, _State) ->
-    emqx_stats:cancel_update(cm_stats).
+    emqx_stats:cancel_update(conn_stats).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
@@ -168,7 +174,16 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-update_conn_stats() ->
+clean_down({Pid, ClientId}) ->
+    Conn = {ClientId, Pid},
+    case ets:member(?CONN_TAB, ClientId)
+         orelse ets:member(?CONN_ATTRS_TAB, Conn) of
+        true ->
+            do_unregister_connection(Conn);
+        false -> false
+    end.
+
+stats_fun() ->
     case ets:info(?CONN_TAB, size) of
         undefined -> ok;
         Size -> emqx_stats:setstat('connections/count', 'connections/max', Size)

src/emqx_cm_sup.erl

@@ -1,6 +1,5 @@
 %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
-%%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
 %% You may obtain a copy of the License at
@@ -28,13 +27,13 @@ init([]) ->
     Banned = #{id => banned,
                start => {emqx_banned, start_link, []},
                restart => permanent,
-               shutdown => 5000,
+               shutdown => 1000,
                type => worker,
                modules => [emqx_banned]},
     Manager = #{id => manager,
                 start => {emqx_cm, start_link, []},
                 restart => permanent,
-                shutdown => 5000,
+                shutdown => 2000,
                 type => worker,
                 modules => [emqx_cm]},
     {ok, {{one_for_one, 10, 100}, [Banned, Manager]}}.

src/emqx_connection.erl

@@ -16,15 +16,14 @@
 
 -behaviour(gen_server).
 
--define(LOG_HEADER, "[TCP]").
-
 -include("emqx.hrl").
 -include("emqx_mqtt.hrl").
+
+-define(LOG_HEADER, "[MQTT]").
 -include("logger.hrl").
 
 -export([start_link/3]).
--export([info/1, attrs/1]).
+-export([info/1, attrs/1, stats/1]).
--export([stats/1]).
 -export([kick/1]).
 -export([session/1]).
 
@@ -38,19 +37,20 @@
           peername,
           sockname,
           conn_state,
-          await_recv,
+          active_n,
           proto_state,
           parser_state,
+          gc_state,
           keepalive,
           enable_stats,
           stats_timer,
-          incoming,
           rate_limit,
-          publish_limit,
+          pub_limit,
           limit_timer,
           idle_timeout
          }).
 
+-define(DEFAULT_ACTIVE_N, 100).
 -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
 
 start_link(Transport, Socket, Options) ->
@@ -69,17 +69,17 @@ info(#state{transport     = Transport,
             peername    = Peername,
             sockname    = Sockname,
             conn_state  = ConnState,
-            await_recv    = AwaitRecv,
+            active_n    = ActiveN,
             rate_limit  = RateLimit,
-            publish_limit = PubLimit,
+            pub_limit   = PubLimit,
             proto_state = ProtoState}) ->
     ConnInfo = [{socktype, Transport:type(Socket)},
                 {peername, Peername},
                 {sockname, Sockname},
                 {conn_state, ConnState},
-                {await_recv, AwaitRecv},
+                {active_n, ActiveN},
                 {rate_limit, esockd_rate_limit:info(RateLimit)},
-                {publish_limit, esockd_rate_limit:info(PubLimit)}],
+                {pub_limit, esockd_rate_limit:info(PubLimit)}],
     ProtoInfo = emqx_protocol:info(ProtoState),
     lists:usort(lists:append(ConnInfo, ProtoInfo)).
 
@@ -129,6 +129,7 @@ init([Transport, RawSocket, Options]) ->
             Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
             RateLimit = init_limiter(proplists:get_value(rate_limit, Options)),
             PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)),
+            ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N),
             EnableStats = emqx_zone:get_env(Zone, enable_stats, true),
             IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000),
             SendFun = send_fun(Transport, Socket),
@@ -137,22 +138,22 @@ init([Transport, RawSocket, Options]) ->
                                               peercert => Peercert,
                                               sendfun  => SendFun}, Options),
             ParserState = emqx_protocol:parser(ProtoState),
+            GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
+            GcState = emqx_gc:init(GcPolicy),
             State = run_socket(#state{transport    = Transport,
                                       socket       = Socket,
                                       peername     = Peername,
-                                      await_recv    = false,
                                       conn_state   = running,
+                                      active_n     = ActiveN,
                                       rate_limit   = RateLimit,
-                                      publish_limit = PubLimit,
+                                      pub_limit    = PubLimit,
                                       proto_state  = ProtoState,
                                       parser_state = ParserState,
+                                      gc_state     = GcState,
                                       enable_stats = EnableStats,
                                       idle_timeout = IdleTimout
                                      }),
-            GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
-            ok = emqx_gc:init(GcPolicy),
             ok = emqx_misc:init_proc_mng_policy(Zone),
-
             emqx_logger:set_metadata_peername(esockd_net:format(Peername)),
             gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}],
                                   State, self(), IdleTimout);
@@ -205,16 +206,16 @@ handle_cast(Msg, State) ->
 handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) ->
     case emqx_protocol:deliver(PubOrAck, ProtoState) of
         {ok, ProtoState1} ->
-            State1 = ensure_stats_timer(State#state{proto_state = ProtoState1}),
+            State1 = State#state{proto_state = ProtoState1},
-            ok = maybe_gc(State1, PubOrAck),
+            {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))};
-            {noreply, State1};
         {error, Reason} ->
             shutdown(Reason, State)
     end;
+
 handle_info({timeout, Timer, emit_stats},
             State = #state{stats_timer = Timer,
-                           proto_state = ProtoState
+                           proto_state = ProtoState,
-                          }) ->
+                           gc_state = GcState}) ->
     emqx_metrics:commit(),
     emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)),
     NewState = State#state{stats_timer = undefined},
@@ -223,12 +224,14 @@ handle_info({timeout, Timer, emit_stats},
         continue ->
             {noreply, NewState};
         hibernate ->
-            ok = emqx_gc:reset(),
+            %% going to hibernate, reset gc stats
-            {noreply, NewState, hibernate};
+            GcState1 = emqx_gc:reset(GcState),
+            {noreply, NewState#state{gc_state = GcState1}, hibernate};
         {shutdown, Reason} ->
             ?LOG(warning, "shutdown due to ~p", [Reason]),
             shutdown(Reason, NewState)
     end;
+
 handle_info(timeout, State) ->
     shutdown(idle_timeout, State);
 
@@ -243,19 +246,26 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) ->
     ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]),
     shutdown(conflict, State);
 
+handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl ->
+    process_incoming(Data, State);
+
+%% Rate limit here, cool:)
+handle_info({tcp_passive, _Sock}, State) ->
+    {noreply, run_socket(ensure_rate_limit(State))};
+%% FIXME Later
+handle_info({ssl_passive, _Sock}, State) ->
+    {noreply, run_socket(ensure_rate_limit(State))};
+
+handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error ->
+    shutdown(Reason, State);
+
+handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed ->
+    shutdown(closed, State);
+
+%% Rate limit timer
 handle_info(activate_sock, State) ->
     {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})};
 
-handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) ->
-    ?LOG(debug, "RECV ~p", [Data]),
-    Size = iolist_size(Data),
-    emqx_metrics:trans(inc, 'bytes/received', Size),
-    Incoming = #{bytes => Size, packets => 0},
-    handle_packet(Data, State#state{await_recv = false, incoming = Incoming});
-
-handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) ->
-    shutdown(Reason, State);
-
 handle_info({inet_reply, _Sock, ok}, State) ->
     {noreply, State};
 
@@ -310,26 +320,37 @@ code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
 %%------------------------------------------------------------------------------
-%% Parse and handle packets
+%% Internals: process incoming, parse and handle packets
 %%------------------------------------------------------------------------------
 
-%% Receive and parse data
+process_incoming(Data, State) ->
-handle_packet(<<>>, State0) ->
+    Oct = iolist_size(Data),
-    State = ensure_stats_timer(ensure_rate_limit(State0)),
+    ?LOG(debug, "RECV ~p", [Data]),
-    ok = maybe_gc(State, incoming),
+    emqx_pd:update_counter(incoming_bytes, Oct),
+    emqx_metrics:trans(inc, 'bytes/received', Oct),
+    case handle_packet(Data, State) of
+        {noreply, State1} ->
+            State2 = maybe_gc({1, Oct}, State1),
+            {noreply, ensure_stats_timer(State2)};
+        Shutdown -> Shutdown
+    end.
+
+%% Parse and handle packets
+handle_packet(<<>>, State) ->
     {noreply, State};
+
 handle_packet(Data, State = #state{proto_state  = ProtoState,
                                    parser_state = ParserState,
                                    idle_timeout = IdleTimeout}) ->
-    case catch emqx_frame:parse(Data, ParserState) of
+    try emqx_frame:parse(Data, ParserState) of
-        {more, NewParserState} ->
+        {more, ParserState1} ->
-            {noreply, run_socket(State#state{parser_state = NewParserState}), IdleTimeout};
+            {noreply, State#state{parser_state = ParserState1}, IdleTimeout};
         {ok, Packet = ?PACKET(Type), Rest} ->
             emqx_metrics:received(Packet),
+            (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1),
             case emqx_protocol:received(Packet, ProtoState) of
                 {ok, ProtoState1} ->
-                    NewState = State#state{proto_state = ProtoState1},
+                    handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1}));
-                    handle_packet(Rest, inc_publish_cnt(Type, reset_parser(NewState)));
                 {error, Reason} ->
                     ?LOG(error, "Process packet error - ~p", [Reason]),
                     shutdown(Reason, State);
@@ -338,38 +359,32 @@ handle_packet(Data, State = #state{proto_state  = ProtoState,
                 {stop, Error, ProtoState1} ->
                     stop(Error, State#state{proto_state = ProtoState1})
             end;
-        {error, Error} ->
+        {error, Reason} ->
-            ?LOG(error, "Framing error - ~p", [Error]),
+            ?LOG(error, "Parse frame error - ~p", [Reason]),
-            shutdown(Error, State);
+            shutdown(Reason, State)
-        {'EXIT', Reason} ->
+    catch
-            ?LOG(error, "Parse failed for ~p~nError data:~p", [Reason, Data]),
+        _:Error ->
+            ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]),
             shutdown(parse_error, State)
     end.
 
 reset_parser(State = #state{proto_state = ProtoState}) ->
     State#state{parser_state = emqx_protocol:parser(ProtoState)}.
 
-inc_publish_cnt(Type, State = #state{incoming = Incoming = #{packets := Cnt}})
-    when Type == ?PUBLISH; Type == ?SUBSCRIBE ->
-    State#state{incoming = Incoming#{packets := Cnt + 1}};
-inc_publish_cnt(_Type, State) ->
-    State.
-
 %%------------------------------------------------------------------------------
 %% Ensure rate limit
-%%------------------------------------------------------------------------------
 
-ensure_rate_limit(State = #state{rate_limit = Rl, publish_limit = Pl,
+ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl}) ->
-                                 incoming = #{packets := Packets, bytes := Bytes}}) ->
+    Limiters = [{Pl, #state.pub_limit, emqx_pd:reset_counter(incoming_pubs)},
-    ensure_rate_limit([{Pl, #state.publish_limit, Packets},
+                {Rl, #state.rate_limit, emqx_pd:reset_counter(incoming_bytes)}],
-                       {Rl, #state.rate_limit, Bytes}], State).
+    ensure_rate_limit(Limiters, State).
 
 ensure_rate_limit([], State) ->
-    run_socket(State);
+    State;
-ensure_rate_limit([{undefined, _Pos, _Num}|Limiters], State) ->
+ensure_rate_limit([{undefined, _Pos, _Cnt}|Limiters], State) ->
     ensure_rate_limit(Limiters, State);
-ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) ->
+ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) ->
-   case esockd_rate_limit:check(Num, Rl) of
+   case esockd_rate_limit:check(Cnt, Rl) of
        {0, Rl1} ->
            ensure_rate_limit(Limiters, setelement(Pos, State, Rl1));
        {Pause, Rl1} ->
@@ -377,17 +392,26 @@ ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) ->
            setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1)
    end.
 
+%%------------------------------------------------------------------------------
+%% Activate socket
+
 run_socket(State = #state{conn_state = blocked}) ->
     State;
-run_socket(State = #state{await_recv = true}) ->
+
-    State;
+run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) ->
-run_socket(State = #state{transport = Transport, socket = Socket}) ->
+    TrueOrN = case Transport:is_ssl(Socket) of
-    Transport:async_recv(Socket, 0, infinity),
+                  true  -> true; %% Cannot set '{active, N}' for SSL:(
-    State#state{await_recv = true}.
+                  false -> N
+              end,
+    ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])),
+    State.
+
+ensure_ok_or_exit(ok) -> ok;
+ensure_ok_or_exit({error, Reason}) ->
+    self() ! {shutdown, Reason}.
 
 %%------------------------------------------------------------------------------
 %% Ensure stats timer
-%%------------------------------------------------------------------------------
 
 ensure_stats_timer(State = #state{enable_stats = true,
                                   stats_timer = undefined,
@@ -395,18 +419,25 @@ ensure_stats_timer(State = #state{enable_stats = true,
     State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)};
 ensure_stats_timer(State) -> State.
 
+%%------------------------------------------------------------------------------
+%% Maybe GC
+
+maybe_gc(_, State = #state{gc_state = undefined}) ->
+    State;
+maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) ->
+    Oct = iolist_size(Payload),
+    maybe_gc({1, Oct}, State);
+maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) ->
+    {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt),
+    State#state{gc_state = GCSt1};
+maybe_gc(_, State) ->
+    State.
+
+%%------------------------------------------------------------------------------
+%% Shutdown or stop
+
 shutdown(Reason, State) ->
     stop({shutdown, Reason}, State).
 
 stop(Reason, State) ->
     {stop, Reason, State}.
-
-%% For incoming messages, bump gc-stats with packet count and totoal volume
-%% For outgoing messages, only 'publish' type is taken into account.
-maybe_gc(#state{incoming = #{bytes := Oct, packets := Cnt}}, incoming) ->
-    ok = emqx_gc:inc(Cnt, Oct);
-maybe_gc(#state{}, {publish, _PacketId, #message{payload = Payload}}) ->
-    Oct = iolist_size(Payload),
-    ok = emqx_gc:inc(1, Oct);
-maybe_gc(_, _) ->
-    ok.

src/emqx_ctl.erl

@@ -54,15 +54,6 @@ run_command([Cmd | Args]) ->
     run_command(list_to_atom(Cmd), Args).
 
 -spec(run_command(cmd(), [string()]) -> ok | {error, term()}).
-% run_command(set, []) ->
-%     emqx_mgmt_cli_cfg:set_usage(), ok;
-
-% run_command(set, Args) ->
-%     emqx_mgmt_cli_cfg:run(["config" | Args]), ok;
-
-% run_command(show, Args) ->
-%     emqx_mgmt_cli_cfg:run(["config" | Args]), ok;
-
 run_command(help, []) ->
     usage();
 run_command(Cmd, Args) when is_atom(Cmd) ->
@@ -96,7 +87,7 @@ usage() ->
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    _ = emqx_tables:new(?TAB, [ordered_set, protected]),
+    ok = emqx_tables:new(?TAB, [protected, ordered_set]),
     {ok, #state{seq = 0}}.
 
 handle_call(Req, _From, State) ->
@@ -160,4 +151,3 @@ register_command_test_() ->
     }.
 
 -endif.
-

src/emqx_gc.erl

@@ -21,74 +21,83 @@
 
 -module(emqx_gc).
 
--export([init/1, inc/2, reset/0]).
+-export([init/1, run/3, info/1, reset/1]).
 
--type st() :: #{ cnt => {integer(), integer()}
+-type(opts() :: #{count => integer(),
-               , oct => {integer(), integer()}
+                  bytes => integer()}).
-               }.
+
+-type(st() :: #{cnt => {integer(), integer()},
+                oct => {integer(), integer()}}).
+
+-type(gc_state() :: {?MODULE, st()}).
 
 -define(disabled, disabled).
 -define(ENABLED(X), (is_integer(X) andalso X > 0)).
 
-%% @doc Initialize force GC parameters.
+%% @doc Initialize force GC state.
--spec init(false | map()) -> ok.
+-spec(init(opts() | false) -> gc_state() | undefined).
 init(#{count := Count, bytes := Bytes}) ->
     Cnt = [{cnt, {Count, Count}} || ?ENABLED(Count)],
     Oct = [{oct, {Bytes, Bytes}} || ?ENABLED(Bytes)],
-    erlang:put(?MODULE, maps:from_list(Cnt ++ Oct)),
+    {?MODULE, maps:from_list(Cnt ++ Oct)};
-    ok;
+init(false) -> undefined.
-init(_) -> erlang:put(?MODULE, #{}), ok.
 
-%% @doc Increase count and bytes stats in one call,
+%% @doc Try to run GC based on reduntions of count or bytes.
-%% ensure gc is triggered at most once, even if both thresholds are hit.
+-spec(run(pos_integer(), pos_integer(), gc_state()) -> {boolean(), gc_state()}).
--spec inc(pos_integer(), pos_integer()) -> ok.
+run(Cnt, Oct, {?MODULE, St}) ->
-inc(Cnt, Oct) ->
+    {Res, St1} = run([{cnt, Cnt}, {oct, Oct}], St),
-    mutate_pd_with(fun(St) -> inc(St, Cnt, Oct) end).
+    {Res, {?MODULE, St1}};
+run(_Cnt, _Oct, undefined) ->
+    {false, undefined}.
 
-%% @doc Reset counters to zero.
+run([], St) ->
--spec reset() -> ok.
+    {false, St};
-reset() ->
+run([{K, N}|T], St) ->
-    mutate_pd_with(fun(St) -> reset(St) end).
+    case dec(K, N, St) of
-
+        {true, St1} ->
-%% ======== Internals ========
+            {true, do_gc(St1)};
-
-%% mutate gc stats numbers in process dict with the given function
-mutate_pd_with(F) ->
-    St = F(erlang:get(?MODULE)),
-    erlang:put(?MODULE, St),
-    ok.
-
-%% Increase count and bytes stats in one call,
-%% ensure gc is triggered at most once, even if both thresholds are hit.
--spec inc(st(), pos_integer(), pos_integer()) -> st().
-inc(St0, Cnt, Oct) ->
-    case do_inc(St0, cnt, Cnt) of
-        {true, St} ->
-            St;
         {false, St1} ->
-            {_, St} = do_inc(St1, oct, Oct),
+            run(T, St1)
-            St
     end.
 
-%% Reset counters to zero.
+%% @doc Info of GC state.
-reset(St) -> reset(cnt, reset(oct, St)).
+-spec(info(gc_state()) -> map() | undefined).
+info({?MODULE, St}) ->
+    St;
+info(undefined) ->
+    undefined.
 
--spec do_inc(st(), cnt | oct, pos_integer()) -> {boolean(), st()}.
+%% @doc Reset counters to zero.
-do_inc(St, Key, Num) ->
+-spec(reset(gc_state()) -> gc_state()).
+reset({?MODULE, St}) ->
+    {?MODULE, do_reset(St)};
+reset(undefined) ->
+    undefined.
+
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+-spec(dec(cnt | oct, pos_integer(), st()) -> {boolean(), st()}).
+dec(Key, Num, St) ->
     case maps:get(Key, St, ?disabled) of
         ?disabled ->
             {false, St};
         {Init, Remain} when Remain > Num ->
             {false, maps:put(Key, {Init, Remain - Num}, St)};
         _ ->
-            {true, do_gc(St)}
+            {true, St}
     end.
 
 do_gc(St) ->
-    erlang:garbage_collect(),
+    true = erlang:garbage_collect(),
-    reset(St).
+    do_reset(St).
 
-reset(Key, St) ->
+do_reset(St) ->
+    do_reset(cnt, do_reset(oct, St)).
+
+%% Reset counters to zero.
+do_reset(Key, St) ->
     case maps:get(Key, St, ?disabled) of
         ?disabled -> St;
         {Init, _} -> maps:put(Key, {Init, Init}, St)

src/emqx_hooks.erl

@@ -139,7 +139,7 @@ lookup(HookPoint) ->
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    _ = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]),
+    ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]),
     {ok, #{}}.
 
 handle_call({add, HookPoint, Callback = #callback{action = Action}}, _From, State) ->

src/emqx_local_bridge.erl

@@ -61,7 +61,7 @@ init([Pool, Id, Node, Topic, Options]) ->
         true ->
             true = erlang:monitor_node(Node, true),
             Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]),
-            emqx_broker:subscribe(Topic, self(), #{share => Group, qos => ?QOS_0}),
+            emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}),
             State = parse_opts(Options, #state{node = Node, subtopic = Topic}),
             MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len,
                                         store_qos0 => true}),

src/emqx_metrics.erl

@@ -285,7 +285,7 @@ qos_sent(?QOS_2) ->
 
 init([]) ->
     % Create metrics table
-    _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]),
+    ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]),
     lists:foreach(fun new/1, ?BYTES_METRICS ++ ?PACKET_METRICS ++ ?MESSAGE_METRICS),
     {ok, #{}, hibernate}.
 

src/emqx_misc.erl

@@ -19,6 +19,8 @@
 
 -export([init_proc_mng_policy/1, conn_proc_mng_policy/1]).
 
+-export([drain_down/1]).
+
 %% @doc Merge options
 -spec(merge_opts(list(), list()) -> list()).
 merge_opts(Defaults, Options) ->
@@ -108,3 +110,19 @@ is_enabled(Max) -> is_integer(Max) andalso Max > ?DISABLED.
 proc_info(Key) ->
     {Key, Value} = erlang:process_info(self(), Key),
     Value.
+
+-spec(drain_down(pos_integer()) -> list(pid())).
+drain_down(Cnt) when Cnt > 0 ->
+    drain_down(Cnt, []).
+
+drain_down(0, Acc) ->
+    lists:reverse(Acc);
+
+drain_down(Cnt, Acc) ->
+    receive
+        {'DOWN', _MRef, process, Pid, _Reason} ->
+            drain_down(Cnt - 1, [Pid|Acc])
+    after 0 ->
+          lists:reverse(Acc)
+    end.
+

src/emqx_mqueue.erl

@@ -5,7 +5,8 @@
 %% You may obtain a copy of the License at
 %%
 %%     http://www.apache.org/licenses/LICENSE-2.0
-%%%% Unless required by applicable law or agreed to in writing, software
+%%
+%% Unless required by applicable law or agreed to in writing, software
 %% distributed under the License is distributed on an "AS IS" BASIS,
 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %% See the License for the specific language governing permissions and

src/emqx_pd.erl

@@ -0,0 +1,33 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+%% @doc The utility functions for erlang process dictionary.
+-module(emqx_pd).
+
+-export([update_counter/2, get_counter/1, reset_counter/1]).
+
+-type(key() :: term()).
+
+-spec(update_counter(key(), number()) -> undefined | number()).
+update_counter(Key, Inc) ->
+    put(Key, get_counter(Key) + Inc).
+
+-spec(get_counter(key()) -> number()).
+get_counter(Key) ->
+    case get(Key) of undefined -> 0; Cnt -> Cnt end.
+
+-spec(reset_counter(key()) -> number()).
+reset_counter(Key) ->
+    case put(Key, 0) of undefined -> 0; Cnt -> Cnt end.
+

src/emqx_plugins.erl

@@ -1,18 +1,16 @@
-%%%===================================================================
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved.
+%%
-%%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
-%%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
-%%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
-%%% You may obtain a copy of the License at
+%%
-%%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
-%%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
-%%%
+%% Unless required by applicable law or agreed to in writing, software
-%%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
-%%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
-%%% See the License for the specific language governing permissions and
+%% limitations under the License.
-%%% limitations under the License.
-%%%===================================================================
 
 -module(emqx_plugins).
 

src/emqx_pmon.erl

@@ -14,24 +14,27 @@
 
 -module(emqx_pmon).
 
+-compile({no_auto_import, [monitor/3]}).
+
 -export([new/0]).
 -export([monitor/2, monitor/3]).
 -export([demonitor/2]).
 -export([find/2]).
--export([erase/2]).
+-export([erase/2, erase_all/2]).
-
+-export([count/1]).
--compile({no_auto_import,[monitor/3]}).
 
 -type(pmon() :: {?MODULE, map()}).
 -export_type([pmon/0]).
 
 -spec(new() -> pmon()).
-new() -> {?MODULE, maps:new()}.
+new() ->
+    {?MODULE, maps:new()}.
 
 -spec(monitor(pid(), pmon()) -> pmon()).
 monitor(Pid, PM) ->
-    monitor(Pid, undefined, PM).
+    ?MODULE:monitor(Pid, undefined, PM).
 
+-spec(monitor(pid(), term(), pmon()) -> pmon()).
 monitor(Pid, Val, {?MODULE, PM}) ->
     {?MODULE, case maps:is_key(Pid, PM) of
                   true  -> PM;
@@ -43,21 +46,36 @@ monitor(Pid, Val, {?MODULE, PM}) ->
 demonitor(Pid, {?MODULE, PM}) ->
     {?MODULE, case maps:find(Pid, PM) of
                   {ok, {Ref, _Val}} ->
-                      %% Don't flush
+                      %% flush
-                      _ = erlang:demonitor(Ref),
+                      _ = erlang:demonitor(Ref, [flush]),
                       maps:remove(Pid, PM);
                   error -> PM
               end}.
 
--spec(find(pid(), pmon()) -> undefined | term()).
+-spec(find(pid(), pmon()) -> error | {ok, term()}).
 find(Pid, {?MODULE, PM}) ->
     case maps:find(Pid, PM) of
         {ok, {_Ref, Val}} ->
-            Val;
+            {ok, Val};
-        error -> undefined
+        error -> error
     end.
 
 -spec(erase(pid(), pmon()) -> pmon()).
 erase(Pid, {?MODULE, PM}) ->
     {?MODULE, maps:remove(Pid, PM)}.
 
+-spec(erase_all([pid()], pmon()) -> {[{pid(), term()}], pmon()}).
+erase_all(Pids, PMon0) ->
+    lists:foldl(
+      fun(Pid, {Acc, PMon}) ->
+          case find(Pid, PMon) of
+              {ok, Val} ->
+                  {[{Pid, Val}|Acc], erase(Pid, PMon)};
+              error -> {Acc, PMon}
+          end
+      end, {[], PMon0}, Pids).
+
+-spec(count(pmon()) -> non_neg_integer()).
+count({?MODULE, PM}) ->
+    maps:size(PM).
+

src/emqx_pool_sup.erl

@@ -37,9 +37,8 @@ spec(ChildId, Args) ->
 start_link(Pool, Type, MFA) ->
     start_link(Pool, Type, emqx_vm:schedulers(), MFA).
 
--spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, term()}).
+-spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa())
-start_link(Pool, Type, Size, MFA) when is_atom(Pool) ->
+      -> {ok, pid()} | {error, term()}).
-    supervisor:start_link({local, Pool}, ?MODULE, [Pool, Type, Size, MFA]);
 start_link(Pool, Type, Size, MFA) ->
     supervisor:start_link(?MODULE, [Pool, Type, Size, MFA]).
 

src/emqx_protocol.erl

@@ -65,7 +65,8 @@
           send_stats,
           connected,
           connected_at,
-          ignore_loop
+          ignore_loop,
+          topic_alias_maximum
         }).
 
 -type(state() :: #pstate{}).
@@ -107,7 +108,8 @@ init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options)
             recv_stats          = #{msg => 0, pkt => 0},
             send_stats          = #{msg => 0, pkt => 0},
             connected           = false,
-            ignore_loop     = emqx_config:get_env(mqtt_ignore_loop_deliver, false)}.
+            ignore_loop         = emqx_config:get_env(mqtt_ignore_loop_deliver, false),
+            topic_alias_maximum = #{to_client => 0, from_client => 0}}.
 
 init_username(Peercert, Options) ->
     case proplists:get_value(peer_cert_as_username, Options) of
@@ -218,8 +220,12 @@ received(Packet = ?PACKET(Type), PState) ->
     trace(recv, Packet),
     try emqx_packet:validate(Packet) of
         true ->
-            {Packet1, PState2} = preprocess_properties(Packet, PState1),
+            case preprocess_properties(Packet, PState1) of
+                {error, ReasonCode} ->
+                    {error, ReasonCode, PState1};
+                {Packet1, PState2} ->
                     process_packet(Packet1, inc_stats(recv, Type, PState2))
+            end
     catch
         error : protocol_error ->
             deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState1),
@@ -244,6 +250,13 @@ received(Packet = ?PACKET(Type), PState) ->
 %%------------------------------------------------------------------------------
 %% Preprocess MQTT Properties
 %%------------------------------------------------------------------------------
+preprocess_properties(Packet = #mqtt_packet{
+                                   variable = #mqtt_packet_connect{
+                                                  properties = #{'Topic-Alias-Maximum' := ToClient}
+                                              }
+                               },
+                      PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) ->
+    {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}};
 
 %% Subscription Identifier
 preprocess_properties(Packet = #mqtt_packet{
@@ -257,22 +270,46 @@ preprocess_properties(Packet = #mqtt_packet{
     {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState};
 
 %% Topic Alias Mapping
+preprocess_properties(#mqtt_packet{
+                          variable = #mqtt_packet_publish{
+                              properties = #{'Topic-Alias' := 0}}
+                      },
+                      PState) ->
+    deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState),
+    {error, ?RC_TOPIC_ALIAS_INVALID};
+
 preprocess_properties(Packet = #mqtt_packet{
                                   variable = Publish = #mqtt_packet_publish{
                                                           topic_name = <<>>,
                                                           properties = #{'Topic-Alias' := AliasId}}
                                  },
-                      PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) ->
+                      PState = #pstate{proto_ver = ?MQTT_PROTO_V5, 
+                                       topic_aliases = Aliases, 
+                                       topic_alias_maximum = #{from_client := TopicAliasMaximum}}) ->
+    case AliasId =< TopicAliasMaximum of
+        true ->
             {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{
                                                topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState};
+        false ->
+            deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState),
+            {error, ?RC_TOPIC_ALIAS_INVALID}
+    end;
 
 preprocess_properties(Packet = #mqtt_packet{
                                    variable = #mqtt_packet_publish{
                                                   topic_name = Topic,
                                                   properties = #{'Topic-Alias' := AliasId}}
                                },
-                      PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) ->
+                      PState = #pstate{proto_ver = ?MQTT_PROTO_V5, 
+                                       topic_aliases = Aliases,
+                                       topic_alias_maximum = #{from_client := TopicAliasMaximum}}) ->
+    case AliasId =< TopicAliasMaximum of
+        true ->
             {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}};
+        false ->
+            deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState),
+            {error, ?RC_TOPIC_ALIAS_INVALID}
+    end;
 
 preprocess_properties(Packet, PState) ->
     {Packet, PState}.
@@ -280,7 +317,6 @@ preprocess_properties(Packet, PState) ->
 %%------------------------------------------------------------------------------
 %% Process MQTT Packet
 %%------------------------------------------------------------------------------
-
 process_packet(?CONNECT_PACKET(
                   #mqtt_packet_connect{proto_name  = ProtoName,
                                        proto_ver   = ProtoVer,
@@ -308,6 +344,7 @@ process_packet(?CONNECT_PACKET(
                                          conn_props   = ConnProps,
                                          is_bridge    = IsBridge,
                                          connected_at = os:timestamp()}),
+
     connack(
       case check_connect(ConnPkt, PState1) of
           {ok, PState2} ->
@@ -321,7 +358,8 @@ process_packet(?CONNECT_PACKET(
                       case try_open_session(PState3) of
                           {ok, SPid, SP} ->
                               PState4 = PState3#pstate{session = SPid, connected = true},
-                              ok = emqx_cm:register_connection(client_id(PState4), attrs(PState4)),
+                              ok = emqx_cm:register_connection(client_id(PState4)),
+                              true = emqx_cm:set_conn_attrs(client_id(PState4), attrs(PState4)),
                               %% Start keepalive
                               start_keepalive(Keepalive, PState4),
                               %% Success
@@ -507,18 +545,18 @@ do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId),
 
 puback(?QOS_0, _PacketId, _Result, PState) ->
     {ok, PState};
+puback(?QOS_1, PacketId, [], PState) ->
+    deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState);
+puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch?
+    deliver({puback, PacketId, ?RC_SUCCESS}, PState);
 puback(?QOS_1, PacketId, {error, ReasonCode}, PState) ->
     deliver({puback, PacketId, ReasonCode}, PState);
-puback(?QOS_1, PacketId, {ok, []}, PState) ->
+puback(?QOS_2, PacketId, [], PState) ->
-    deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState);
-puback(?QOS_1, PacketId, {ok, _}, PState) ->
-    deliver({puback, PacketId, ?RC_SUCCESS}, PState);
-puback(?QOS_2, PacketId, {error, ReasonCode}, PState) ->
-    deliver({pubrec, PacketId, ReasonCode}, PState);
-puback(?QOS_2, PacketId, {ok, []}, PState) ->
     deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState);
-puback(?QOS_2, PacketId, {ok, _}, PState) ->
+puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch?
-    deliver({pubrec, PacketId, ?RC_SUCCESS}, PState).
+    deliver({pubrec, PacketId, ?RC_SUCCESS}, PState);
+puback(?QOS_2, PacketId, {error, ReasonCode}, PState) ->
+    deliver({pubrec, PacketId, ReasonCode}, PState).
 
 %%------------------------------------------------------------------------------
 %% Deliver Packet -> Client
@@ -532,7 +570,8 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone,
                                                      proto_ver = ?MQTT_PROTO_V5,
                                                      client_id = ClientId,
                                                      conn_props = ConnProps,
-                                                     is_assigned = IsAssigned}) ->
+                                                     is_assigned = IsAssigned,
+                                                     topic_alias_maximum = TopicAliasMaximum}) ->
     ResponseInformation = case maps:find('Request-Response-Information', ConnProps) of
                               {ok, 1} ->
                                   iolist_to_binary(emqx_config:get_env(response_topic_prefix));
@@ -570,16 +609,19 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone,
                  undefined -> Props2;
                  Keepalive -> Props2#{'Server-Keep-Alive' => Keepalive}
              end,
-    send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState);
+
+    PState1 = PState#pstate{topic_alias_maximum = TopicAliasMaximum#{from_client => MaxAlias}},
+
+    send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState1);
 
 deliver({connack, ReasonCode, SP}, PState) ->
     send(?CONNACK_PACKET(ReasonCode, SP), PState);
 
-deliver({publish, PacketId, Msg}, PState = #pstate{mountpoint = MountPoint}) ->
+deliver({publish, PacketId, Msg = #message{headers = Headers}}, PState = #pstate{mountpoint = MountPoint}) ->
     _ = emqx_hooks:run('message.delivered', [credentials(PState)], Msg),
     Msg1 = emqx_message:update_expiry(Msg),
     Msg2 = emqx_mountpoint:unmount(MountPoint, Msg1),
-    send(emqx_packet:from_message(PacketId, Msg2), PState);
+    send(emqx_packet:from_message(PacketId, Msg2#message{headers = maps:remove('Topic-Alias', Headers)}), PState);
     
 deliver({puback, PacketId, ReasonCode}, PState) ->
     send(?PUBACK_PACKET(PacketId, ReasonCode), PState);
@@ -629,14 +671,12 @@ maybe_use_username_as_clientid(ClientId, undefined, _PState) ->
     ClientId;
 maybe_use_username_as_clientid(ClientId, Username, #pstate{zone = Zone}) ->
     case emqx_zone:get_env(Zone, use_username_as_clientid, false) of
-        true ->
+        true -> Username;
-            Username;
+        false -> ClientId
-        false ->
-            ClientId
     end.
 
 %%------------------------------------------------------------------------------
-%% Assign a clientid
+%% Assign a clientId
 
 maybe_assign_client_id(PState = #pstate{client_id = <<>>, ack_props = AckProps}) ->
     ClientId = emqx_guid:to_base62(emqx_guid:gen()),
@@ -660,41 +700,37 @@ try_open_session(PState = #pstate{zone        = Zone,
         clean_start => CleanStart,
         will_msg    => WillMsg
     },
-    SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}, {topic_alias_maximum, PState}]),
+    SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}]),
     case emqx_sm:open_session(SessAttrs1) of
         {ok, SPid} ->
             {ok, SPid, false};
         Other -> Other
     end.
 
-set_session_attrs({max_inflight, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) ->
-    maps:put(max_inflight, if
-                               ProtoVer =:= ?MQTT_PROTO_V5 ->
-                                   get_property('Receive-Maximum', ConnProps, 65535);
-                               true ->
-                                   emqx_zone:get_env(Zone, max_inflight, 65535)
-                           end, SessAttrs);
-set_session_attrs({expiry_interval, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps, clean_start = CleanStart}}, SessAttrs) ->
-    maps:put(expiry_interval, if
-                               ProtoVer =:= ?MQTT_PROTO_V5 ->
-                                   get_property('Session-Expiry-Interval', ConnProps, 0);
-                               true ->
-                                   case CleanStart of
-                                       true -> 0;
-                                       false ->
-                                           emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff)
-                                   end
-                              end, SessAttrs);
-set_session_attrs({topic_alias_maximum, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) ->
-    maps:put(topic_alias_maximum, if
-                                    ProtoVer =:= ?MQTT_PROTO_V5 ->
-                                        get_property('Topic-Alias-Maximum', ConnProps, 0);
-                                    true ->
-                                        emqx_zone:get_env(Zone, max_topic_alias, 0)
-                                  end, SessAttrs);
-set_session_attrs({_, #pstate{}}, SessAttrs) ->
-    SessAttrs.
 
+set_session_attrs({max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) ->
+    maps:put(max_inflight, get_property('Receive-Maximum', ConnProps, 65535), SessAttrs);
+
+set_session_attrs({max_inflight, #pstate{zone = Zone}}, SessAttrs) ->
+    maps:put(max_inflight, emqx_zone:get_env(Zone, max_inflight, 65535), SessAttrs);
+
+set_session_attrs({expiry_interval, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) ->
+    maps:put(expiry_interval, get_property('Session-Expiry-Interval', ConnProps, 0), SessAttrs);
+
+set_session_attrs({expiry_interval, #pstate{zone = Zone, clean_start = CleanStart}}, SessAttrs) ->
+    maps:put(expiry_interval, case CleanStart of
+                                  true -> 0;
+                                  false -> emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff)
+                              end, SessAttrs);
+
+set_session_attrs({topic_alias_maximum, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) ->
+    maps:put(topic_alias_maximum, get_property('Topic-Alias-Maximum', ConnProps, 0), SessAttrs);
+
+set_session_attrs({topic_alias_maximum, #pstate{zone = Zone}}, SessAttrs) ->
+    maps:put(topic_alias_maximum, emqx_zone:get_env(Zone, max_topic_alias, 0), SessAttrs);
+
+set_session_attrs(_, SessAttrs) ->
+    SessAttrs.
 
 authenticate(Credentials, Password) ->
     case emqx_access_control:authenticate(Credentials, Password) of
@@ -803,12 +839,6 @@ check_publish(Packet, PState) ->
     run_check_steps([fun check_pub_caps/2,
                      fun check_pub_acl/2], Packet, PState).
 
-check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain},
-                            variable = #mqtt_packet_publish{
-                                          properties = #{'Topic-Alias' := TopicAlias}
-                                         }},
-               #pstate{zone = Zone}) ->
-    emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias});
 check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain},
                             variable = #mqtt_packet_publish{ properties = _Properties}},
                #pstate{zone = Zone}) ->
@@ -881,14 +911,13 @@ shutdown(_Reason, #pstate{client_id = undefined}) ->
     ok;
 shutdown(_Reason, #pstate{connected = false}) ->
     ok;
-shutdown(Reason, #pstate{client_id = ClientId}) when Reason =:= conflict;
+shutdown(conflict, _PState) ->
-                                                     Reason =:= discard ->
+    ok;
-    emqx_cm:unregister_connection(ClientId);
+shutdown(discard, _PState) ->
-shutdown(Reason, PState = #pstate{connected = true,
+    ok;
-                                  client_id = ClientId}) ->
+shutdown(Reason, PState) ->
     ?LOG(info, "Shutdown for ~p", [Reason]),
-    emqx_hooks:run('client.disconnected', [credentials(PState), Reason]),
+    emqx_hooks:run('client.disconnected', [credentials(PState), Reason]).
-    emqx_cm:unregister_connection(ClientId).
 
 start_keepalive(0, _PState) ->
     ignore;

src/emqx_rate_limiter.erl

@@ -1,18 +1,16 @@
-%%%-------------------------------------------------------------------
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io)
+%%
-%%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
-%%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
-%%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
-%%% You may obtain a copy of the License at
+%%
-%%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
-%%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
-%%%
+%% Unless required by applicable law or agreed to in writing, software
-%%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
-%%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
-%%% See the License for the specific language governing permissions and
+%% limitations under the License.
-%%% limitations under the License.
-%%%-------------------------------------------------------------------
 
 -module(emqx_rate_limiter).
 

src/emqx_router.erl

@@ -28,23 +28,22 @@
 -export([start_link/2]).
 
 %% Route APIs
--export([add_route/1, add_route/2, add_route/3]).
+-export([add_route/1, add_route/2]).
--export([get_routes/1]).
+-export([do_add_route/1, do_add_route/2]).
--export([del_route/1, del_route/2, del_route/3]).
+-export([match_routes/1, lookup_routes/1, has_routes/1]).
--export([has_routes/1, match_routes/1, print_routes/1]).
+-export([delete_route/1, delete_route/2]).
+-export([do_delete_route/1, do_delete_route/2]).
+-export([print_routes/1]).
 -export([topics/0]).
+
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
--type(destination() :: node() | {binary(), node()}).
+-type(group() :: binary()).
-
+-type(destination() :: node() | {group(), node()}).
--record(batch, {enabled, timer, pending}).
--record(state, {pool, id, batch :: #batch{}}).
 
 -define(ROUTE, emqx_route).
--define(BATCH(Enabled), #batch{enabled = Enabled}).
--define(BATCH(Enabled, Pending), #batch{enabled = Enabled, pending = Pending}).
 
 %%------------------------------------------------------------------------------
 %% Mnesia bootstrap
@@ -62,10 +61,10 @@ mnesia(copy) ->
     ok = ekka_mnesia:copy_table(?ROUTE).
 
 %%------------------------------------------------------------------------------
-%% Strat a router
+%% Start a router
 %%------------------------------------------------------------------------------
 
--spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}).
+-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()).
 start_link(Pool, Id) ->
     gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)},
                           ?MODULE, [Pool, Id], [{hibernate_after, 1000}]).
@@ -74,51 +73,69 @@ start_link(Pool, Id) ->
 %% Route APIs
 %%------------------------------------------------------------------------------
 
--spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok).
+-spec(add_route(emqx_topic:topic()) -> ok | {error, term()}).
 add_route(Topic) when is_binary(Topic) ->
-    add_route(#route{topic = Topic, dest = node()});
+    add_route(Topic, node()).
-add_route(Route = #route{topic = Topic}) ->
-    cast(pick(Topic), {add_route, Route}).
 
--spec(add_route(emqx_topic:topic(), destination()) -> ok).
+-spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}).
 add_route(Topic, Dest) when is_binary(Topic) ->
-    add_route(#route{topic = Topic, dest = Dest}).
+    call(pick(Topic), {add_route, Topic, Dest}).
 
--spec(add_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok).
+-spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}).
-add_route(From, Topic, Dest) when is_binary(Topic) ->
+do_add_route(Topic) when is_binary(Topic) ->
-    cast(pick(Topic), {add_route, From, #route{topic = Topic, dest = Dest}}).
+    do_add_route(Topic, node()).
 
--spec(get_routes(emqx_topic:topic()) -> [emqx_types:route()]).
+-spec(do_add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}).
-get_routes(Topic) ->
+do_add_route(Topic, Dest) when is_binary(Topic) ->
+    Route = #route{topic = Topic, dest = Dest},
+    case lists:member(Route, lookup_routes(Topic)) of
+        true  -> ok;
+        false ->
+            ok = emqx_router_helper:monitor(Dest),
+            case emqx_topic:wildcard(Topic) of
+                true  -> trans(fun insert_trie_route/1, [Route]);
+                false -> insert_direct_route(Route)
+            end
+    end.
+
+%% @doc Match routes
+-spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]).
+match_routes(Topic) when is_binary(Topic) ->
+    %% Optimize: routing table will be replicated to all router nodes.
+    Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]),
+    lists:append([lookup_routes(To) || To <- [Topic | Matched]]).
+
+-spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]).
+lookup_routes(Topic) ->
     ets:lookup(?ROUTE, Topic).
 
--spec(del_route(emqx_topic:topic() | emqx_types:route()) -> ok).
-del_route(Topic) when is_binary(Topic) ->
-    del_route(#route{topic = Topic, dest = node()});
-del_route(Route = #route{topic = Topic}) ->
-    cast(pick(Topic), {del_route, Route}).
-
--spec(del_route(emqx_topic:topic(), destination()) -> ok).
-del_route(Topic, Dest) when is_binary(Topic) ->
-    del_route(#route{topic = Topic, dest = Dest}).
-
--spec(del_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok).
-del_route(From, Topic, Dest) when is_binary(Topic) ->
-    cast(pick(Topic), {del_route, From, #route{topic = Topic, dest = Dest}}).
-
 -spec(has_routes(emqx_topic:topic()) -> boolean()).
 has_routes(Topic) when is_binary(Topic) ->
     ets:member(?ROUTE, Topic).
 
--spec(topics() -> list(emqx_topic:topic())).
+-spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}).
-topics() -> mnesia:dirty_all_keys(?ROUTE).
+delete_route(Topic) when is_binary(Topic) ->
+    delete_route(Topic, node()).
 
-%% @doc Match routes
+-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}).
-%% Optimize: routing table will be replicated to all router nodes.
+delete_route(Topic, Dest) when is_binary(Topic) ->
--spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]).
+    call(pick(Topic), {delete_route, Topic, Dest}).
-match_routes(Topic) when is_binary(Topic) ->
+
-    Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]),
+-spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}).
-    lists:append([get_routes(To) || To <- [Topic | Matched]]).
+do_delete_route(Topic) when is_binary(Topic) ->
+    do_delete_route(Topic, node()).
+
+-spec(do_delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}).
+do_delete_route(Topic, Dest) ->
+    Route = #route{topic = Topic, dest = Dest},
+    case emqx_topic:wildcard(Topic) of
+        true  -> trans(fun delete_trie_route/1, [Route]);
+        false -> delete_direct_route(Route)
+    end.
+
+-spec(topics() -> list(emqx_topic:topic())).
+topics() ->
+    mnesia:dirty_all_keys(?ROUTE).
 
 %% @doc Print routes to a topic
 -spec(print_routes(emqx_topic:topic()) -> ok).
@@ -127,82 +144,41 @@ print_routes(Topic) ->
                       io:format("~s -> ~s~n", [To, Dest])
                   end, match_routes(Topic)).
 
-cast(Router, Msg) ->
+call(Router, Msg) ->
-    gen_server:cast(Router, Msg).
+    gen_server:call(Router, Msg, infinity).
 
 pick(Topic) ->
-    gproc_pool:pick_worker(router, Topic).
+    gproc_pool:pick_worker(router_pool, Topic).
 
 %%------------------------------------------------------------------------------
 %% gen_server callbacks
 %%------------------------------------------------------------------------------
 
 init([Pool, Id]) ->
-    rand:seed(exsplus, erlang:timestamp()),
+    true = gproc_pool:connect_worker(Pool, {Pool, Id}),
-    gproc_pool:connect_worker(Pool, {Pool, Id}),
+    {ok, #{pool => Pool, id => Id}}.
-    Batch = #batch{enabled = emqx_config:get_env(route_batch_clean, false),
+
-                   pending = sets:new()},
+handle_call({add_route, Topic, Dest}, _From, State) ->
-    {ok, ensure_batch_timer(#state{pool = Pool, id = Id, batch = Batch})}.
+    Ok = do_add_route(Topic, Dest),
+    {reply, Ok, State};
+
+handle_call({delete_route, Topic, Dest}, _From, State) ->
+    Ok = do_delete_route(Topic, Dest),
+    {reply, Ok, State};
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[Router] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({add_route, From, Route}, State) ->
-    {noreply, NewState} = handle_cast({add_route, Route}, State),
-    _ = gen_server:reply(From, ok),
-    {noreply, NewState};
-
-handle_cast({add_route, Route = #route{topic = Topic, dest = Dest}}, State) ->
-    case lists:member(Route, get_routes(Topic)) of
-        true  -> ok;
-        false ->
-            ok = emqx_router_helper:monitor(Dest),
-            case emqx_topic:wildcard(Topic) of
-                true  -> log(trans(fun add_trie_route/1, [Route]));
-                false -> add_direct_route(Route)
-            end
-    end,
-    {noreply, State};
-
-handle_cast({del_route, From, Route}, State) ->
-    {noreply, NewState} = handle_cast({del_route, Route}, State),
-    _ = gen_server:reply(From, ok),
-    {noreply, NewState};
-
-handle_cast({del_route, Route = #route{topic = Topic, dest = Dest}}, State) when is_tuple(Dest) ->
-    {noreply, case emqx_topic:wildcard(Topic) of
-                  true  -> log(trans(fun del_trie_route/1, [Route])),
-                           State;
-                  false -> del_direct_route(Route, State)
-              end};
-
-handle_cast({del_route, Route = #route{topic = Topic}}, State) ->
-    %% Confirm if there are still subscribers...
-    {noreply, case ets:member(emqx_subscriber, Topic) of
-                  true  -> State;
-                  false ->
-                      case emqx_topic:wildcard(Topic) of
-                          true  -> log(trans(fun del_trie_route/1, [Route])),
-                                   State;
-                          false -> del_direct_route(Route, State)
-                      end
-              end};
-
 handle_cast(Msg, State) ->
     emqx_logger:error("[Router] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({timeout, _TRef, batch_delete}, State = #state{batch = Batch}) ->
-    _ = del_direct_routes(sets:to_list(Batch#batch.pending)),
-    {noreply, ensure_batch_timer(State#state{batch = ?BATCH(true, sets:new())}), hibernate};
-
 handle_info(Info, State) ->
     emqx_logger:error("[Router] unexpected info: ~p", [Info]),
     {noreply, State}.
 
-terminate(_Reason, #state{pool = Pool, id = Id, batch = Batch}) ->
+terminate(_Reason, #{pool := Pool, id := Id}) ->
-    _ = cacel_batch_timer(Batch),
     gproc_pool:disconnect_worker(Pool, {Pool, Id}).
 
 code_change(_OldVsn, State, _Extra) ->
@@ -212,50 +188,23 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-ensure_batch_timer(State = #state{batch = #batch{enabled = false}}) ->
+insert_direct_route(Route) ->
-    State;
-ensure_batch_timer(State = #state{batch = Batch}) ->
-    TRef = erlang:start_timer(50 + rand:uniform(50), self(), batch_delete),
-    State#state{batch = Batch#batch{timer = TRef}}.
-
-cacel_batch_timer(#batch{enabled = false}) ->
-    ok;
-cacel_batch_timer(#batch{enabled = true, timer = TRef}) ->
-    catch erlang:cancel_timer(TRef).
-
-add_direct_route(Route) ->
     mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]).
 
-add_trie_route(Route = #route{topic = Topic}) ->
+insert_trie_route(Route = #route{topic = Topic}) ->
     case mnesia:wread({?ROUTE, Topic}) of
         [] -> emqx_trie:insert(Topic);
         _  -> ok
     end,
     mnesia:write(?ROUTE, Route, sticky_write).
 
-del_direct_route(Route, State = #state{batch = ?BATCH(false)}) ->
+delete_direct_route(Route) ->
-    del_direct_route(Route), State;
-del_direct_route(Route, State = #state{batch = Batch = ?BATCH(true, Pending)}) ->
-    State#state{batch = Batch#batch{pending = sets:add_element(Route, Pending)}}.
-
-del_direct_route(Route) ->
     mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]).
 
-del_direct_routes([]) ->
+delete_trie_route(Route = #route{topic = Topic}) ->
-    ok;
-del_direct_routes(Routes) ->
-    DelFun = fun(R = #route{topic = Topic}) ->
-                 case ets:member(emqx_subscriber, Topic) of
-                     true  -> ok;
-                     false -> mnesia:delete_object(?ROUTE, R, sticky_write)
-                 end
-             end,
-    mnesia:async_dirty(fun lists:foreach/2, [DelFun, Routes]).
-
-del_trie_route(Route = #route{topic = Topic}) ->
     case mnesia:wread({?ROUTE, Topic}) of
         [Route] -> %% Remove route and trie
-                   mnesia:delete_object(?ROUTE, Route, sticky_write),
+                   ok = mnesia:delete_object(?ROUTE, Route, sticky_write),
                    emqx_trie:delete(Topic);
         [_|_]   -> %% Remove route only
                    mnesia:delete_object(?ROUTE, Route, sticky_write);
@@ -266,11 +215,7 @@ del_trie_route(Route = #route{topic = Topic}) ->
 -spec(trans(function(), list(any())) -> ok | {error, term()}).
 trans(Fun, Args) ->
     case mnesia:transaction(Fun, Args) of
-        {atomic, _}      -> ok;
+        {atomic, Ok} -> Ok;
-        {aborted, Error} -> {error, Error}
+        {aborted, Reason} -> {error, Reason}
     end.
 
-log(ok) -> ok;
-log({error, Reason}) ->
-    emqx_logger:error("[Router] mnesia aborted: ~p", [Reason]).
-

src/emqx_router_helper.erl

@@ -31,15 +31,11 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
-%% internal export
+%% Internal export
 -export([stats_fun/0]).
 
 -record(routing_node, {name, const = unused}).
--record(state, {nodes = []}).
 
--compile({no_auto_import, [monitor/1]}).
-
--define(SERVER, ?MODULE).
 -define(ROUTE, emqx_route).
 -define(ROUTING_NODE, emqx_routing_node).
 -define(LOCK, {?MODULE, cleanup_routes}).
@@ -64,9 +60,9 @@ mnesia(copy) ->
 %%------------------------------------------------------------------------------
 
 %% @doc Starts the router helper
--spec(start_link() -> {ok, pid()} | ignore | {error, any()}).
+-spec(start_link() ->  emqx_types:startlink_ret()).
 start_link() ->
-    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
 %% @doc Monitor routing node
 -spec(monitor(node() | {binary(), node()}) -> ok).
@@ -84,18 +80,18 @@ monitor(Node) when is_atom(Node) ->
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    _ = ekka:monitor(membership),
+    ok = ekka:monitor(membership),
-    _ = mnesia:subscribe({table, ?ROUTING_NODE, simple}),
+    {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}),
     Nodes = lists:foldl(
               fun(Node, Acc) ->
                   case ekka:is_member(Node) of
                       true  -> Acc;
-                      false -> _ = erlang:monitor_node(Node, true),
+                      false -> true = erlang:monitor_node(Node, true),
                                [Node | Acc]
                   end
               end, [], mnesia:dirty_all_keys(?ROUTING_NODE)),
-    emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0),
+    ok = emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0),
-    {ok, #state{nodes = Nodes}, hibernate}.
+    {ok, #{nodes => Nodes}, hibernate}.
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[RouterHelper] unexpected call: ~p", [Req]),
@@ -105,24 +101,29 @@ handle_cast(Msg, State) ->
     emqx_logger:error("[RouterHelper] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({mnesia_table_event, {write, #routing_node{name = Node}, _}}, State = #state{nodes = Nodes}) ->
+handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, State = #{nodes := Nodes}) ->
-    emqx_logger:info("[RouterHelper] write routing node: ~s", [Node]),
     case ekka:is_member(Node) orelse lists:member(Node, Nodes) of
         true -> {noreply, State};
-        false -> _ = erlang:monitor_node(Node, true),
+        false ->
-                 {noreply, State#state{nodes = [Node | Nodes]}}
+            true = erlang:monitor_node(Node, true),
+            {noreply, State#{nodes := [Node | Nodes]}}
     end;
 
-handle_info({mnesia_table_event, _Event}, State) ->
+handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) ->
+    %% ignore
     {noreply, State};
 
-handle_info({nodedown, Node}, State = #state{nodes = Nodes}) ->
+handle_info({mnesia_table_event, Event}, State) ->
+    emqx_logger:error("[RouterHelper] unexpected mnesia_table_event: ~p", [Event]),
+    {noreply, State};
+
+handle_info({nodedown, Node}, State = #{nodes := Nodes}) ->
     global:trans({?LOCK, self()},
                  fun() ->
                      mnesia:transaction(fun cleanup_routes/1, [Node])
                  end),
-    mnesia:dirty_delete(?ROUTING_NODE, Node),
+    ok = mnesia:dirty_delete(?ROUTING_NODE, Node),
-    {noreply, State#state{nodes = lists:delete(Node, Nodes)}, hibernate};
+    {noreply, State#{nodes := lists:delete(Node, Nodes)}, hibernate};
 
 handle_info({membership, {mnesia, down, Node}}, State) ->
     handle_info({nodedown, Node}, State);
@@ -134,8 +135,8 @@ handle_info(Info, State) ->
     emqx_logger:error("[RouteHelper] unexpected info: ~p", [Info]),
     {noreply, State}.
 
-terminate(_Reason, #state{}) ->
+terminate(_Reason, _State) ->
-    ekka:unmonitor(membership),
+    ok = ekka:unmonitor(membership),
     emqx_stats:cancel_update(route_stats),
     mnesia:unsubscribe({table, ?ROUTING_NODE, simple}).
 

src/emqx_router_sup.erl

@@ -17,6 +17,7 @@
 -behaviour(supervisor).
 
 -export([start_link/0]).
+
 -export([init/1]).
 
 start_link() ->
@@ -32,8 +33,7 @@ init([]) ->
                modules  => [emqx_router_helper]},
 
     %% Router pool
-    RouterPool = emqx_pool_sup:spec(emqx_router_pool,
+    RouterPool = emqx_pool_sup:spec([router_pool, hash,
-                                    [router, hash, emqx_vm:schedulers(),
                                      {emqx_router, start_link, []}]),
     {ok, {{one_for_all, 0, 1}, [Helper, RouterPool]}}.
 

src/emqx_sequence.erl

@@ -0,0 +1,60 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_sequence).
+
+-export([create/1, nextval/2, currval/2, reclaim/2, delete/1]).
+
+-type(key() :: term()).
+-type(name() :: atom()).
+-type(seqid() :: non_neg_integer()).
+
+-export_type([seqid/0]).
+
+%% @doc Create a sequence.
+-spec(create(name()) -> ok).
+create(Name) ->
+    emqx_tables:new(Name, [public, set, {write_concurrency, true}]).
+
+%% @doc Next value of the sequence.
+-spec(nextval(name(), key()) -> seqid()).
+nextval(Name, Key) ->
+    ets:update_counter(Name, Key, {2, 1}, {Key, 0}).
+
+%% @doc Current value of the sequence.
+-spec(currval(name(), key()) -> seqid()).
+currval(Name, Key) ->
+    try ets:lookup_element(Name, Key, 2)
+    catch
+        error:badarg -> 0
+    end.
+
+%% @doc Reclaim a sequence id.
+-spec(reclaim(name(), key()) -> seqid()).
+reclaim(Name, Key) ->
+    try ets:update_counter(Name, Key, {2, -1, 0, 0}) of
+        0 -> ets:delete_object(Name, {Key, 0}), 0;
+        I -> I
+    catch
+        error:badarg -> 0
+    end.
+
+%% @doc Delete the sequence.
+-spec(delete(name()) -> boolean()).
+delete(Name) ->
+    case ets:info(Name, name) of
+        Name -> ets:delete(Name);
+        undefined -> false
+    end.
+

src/emqx_session.erl

@@ -144,11 +144,12 @@
           %% Enqueue stats
           enqueue_stats = 0,
 
+          %% GC State
+          gc_state,
+
           %% Created at
           created_at :: erlang:timestamp(),
 
-          topic_alias_maximum :: pos_integer(),
-
           will_msg :: emqx:message(),
 
           will_delay_timer :: reference() | undefined
@@ -160,8 +161,6 @@
 
 -export_type([attr/0]).
 
--define(TIMEOUT, 60000).
-
 -define(LOG(Level, Format, Args, _State),
         emqx_logger:Level("[Session] " ++ Format, Args)).
 
@@ -259,13 +258,15 @@ subscribe(SPid, PacketId, Properties, TopicFilters) ->
 
 %% @doc Called by connection processes when publishing messages
 -spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message())
-      -> {ok, emqx_types:deliver_results()}).
+      -> emqx_types:deliver_results() | {error, term()}).
 publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) ->
     %% Publish QoS0 message directly
     emqx_broker:publish(Msg);
+
 publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) ->
     %% Publish QoS1 message directly
     emqx_broker:publish(Msg);
+
 publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) ->
     %% Register QoS2 message packet ID (and timestamp) to session, then publish
     case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of
@@ -277,6 +278,7 @@ publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) ->
 puback(SPid, PacketId) ->
     gen_server:cast(SPid, {puback, PacketId, ?RC_SUCCESS}).
 
+-spec(puback(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) -> ok).
 puback(SPid, PacketId, ReasonCode) ->
     gen_server:cast(SPid, {puback, PacketId, ReasonCode}).
 
@@ -324,7 +326,7 @@ discard(SPid, ByPid) ->
 
 -spec(update_expiry_interval(spid(), timeout()) -> ok).
 update_expiry_interval(SPid, Interval) ->
-    gen_server:cast(SPid, {expiry_interval, Interval}).
+    gen_server:cast(SPid, {update_expiry_interval, Interval}).
 
 -spec(close(spid()) -> ok).
 close(SPid) ->
@@ -341,11 +343,11 @@ init([Parent, #{zone                := Zone,
                 clean_start     := CleanStart,
                 expiry_interval := ExpiryInterval,
                 max_inflight    := MaxInflight,
-                topic_alias_maximum := TopicAliasMaximum,
                 will_msg        := WillMsg}]) ->
-    emqx_logger:set_metadata_client_id(ClientId),
     process_flag(trap_exit, true),
     true = link(ConnPid),
+    emqx_logger:set_metadata_client_id(ClientId),
+    GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
     IdleTimout = get_env(Zone, idle_timeout, 30000),
     State = #state{idle_timeout      = IdleTimout,
                    clean_start       = CleanStart,
@@ -366,15 +368,14 @@ init([Parent, #{zone                := Zone,
                    enable_stats      = get_env(Zone, enable_stats, true),
                    deliver_stats     = 0,
                    enqueue_stats     = 0,
+                   gc_state          = emqx_gc:init(GcPolicy),
                    created_at        = os:timestamp(),
-                   topic_alias_maximum = TopicAliasMaximum,
                    will_msg          = WillMsg
                   },
-    emqx_sm:register_session(ClientId, attrs(State)),
+    ok = emqx_sm:register_session(ClientId, self()),
-    emqx_sm:set_session_stats(ClientId, stats(State)),
+    true = emqx_sm:set_session_attrs(ClientId, attrs(State)),
+    true = emqx_sm:set_session_stats(ClientId, stats(State)),
     emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]),
-    GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
-    ok = emqx_gc:init(GcPolicy),
     ok = emqx_misc:init_proc_mng_policy(Zone),
     ok = proc_lib:init_ack(Parent, {ok, self()}),
     gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State).
@@ -400,51 +401,54 @@ handle_call(stats, _From, State) ->
 
 handle_call({discard, ByPid}, _From, State = #state{conn_pid = undefined}) ->
     ?LOG(warning, "Discarded by ~p", [ByPid], State),
-    {stop, {shutdown, discard}, ok, State};
+    {stop, discarded, ok, State};
 
 handle_call({discard, ByPid}, _From, State = #state{client_id = ClientId, conn_pid = ConnPid}) ->
     ?LOG(warning, "Conn ~p is discarded by ~p", [ConnPid, ByPid], State),
     ConnPid ! {shutdown, discard, {ClientId, ByPid}},
-    {stop, {shutdown, discard}, ok, State};
+    {stop, discarded, ok, State};
 
 %% PUBLISH: This is only to register packetId to session state.
 %% The actual message dispatching should be done by the caller (e.g. connection) process.
 handle_call({register_publish_packet_id, PacketId, Ts}, _From,
             State = #state{awaiting_rel = AwaitingRel}) ->
-    reply(case is_awaiting_full(State) of
+    reply(
+      case is_awaiting_full(State) of
           false ->
               case maps:is_key(PacketId, AwaitingRel) of
                   true ->
                       {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State};
                   false ->
                       State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)},
-                          {ok, ensure_await_rel_timer(State1)}
+                      {ok, ensure_stats_timer(ensure_await_rel_timer(State1))}
               end;
           true ->
-                  emqx_metrics:trans(inc, 'messages/qos2/dropped'),
               ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State),
+              emqx_metrics:trans(inc, 'messages/qos2/dropped'),
               {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State}
       end);
 
 %% PUBREC:
 handle_call({pubrec, PacketId, _ReasonCode}, _From, State = #state{inflight = Inflight}) ->
-    reply(case emqx_inflight:contain(PacketId, Inflight) of
+    reply(
+      case emqx_inflight:contain(PacketId, Inflight) of
           true ->
-                  {ok, acked(pubrec, PacketId, State)};
+              {ok, ensure_stats_timer(acked(pubrec, PacketId, State))};
           false ->
-                  emqx_metrics:trans(inc, 'packets/pubrec/missed'),
               ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State),
+              emqx_metrics:trans(inc, 'packets/pubrec/missed'),
               {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State}
       end);
 
 %% PUBREL:
 handle_call({pubrel, PacketId, _ReasonCode}, _From, State = #state{awaiting_rel = AwaitingRel}) ->
-    reply(case maps:take(PacketId, AwaitingRel) of
+    reply(
+      case maps:take(PacketId, AwaitingRel) of
           {_Ts, AwaitingRel1} ->
-                  {ok, State#state{awaiting_rel = AwaitingRel1}};
+              {ok, ensure_stats_timer(State#state{awaiting_rel = AwaitingRel1})};
           error ->
+              ?LOG(warning, "The PUBREL PacketId ~w is not found", [PacketId], State),
               emqx_metrics:trans(inc, 'packets/pubrel/missed'),
-                  ?LOG(warning, "Cannot find PUBREL: ~w", [PacketId], State),
               {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State}
       end);
 
@@ -465,7 +469,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}},
                                                   emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]),
                                                   SubMap;
                                               {ok, _SubOpts} ->
-                                                  emqx_broker:set_subopts(Topic, {self(), ClientId}, SubOpts),
+                                                  emqx_broker:set_subopts(Topic, SubOpts),
                                                   %% Why???
                                                   emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]),
                                                   maps:put(Topic, SubOpts, SubMap);
@@ -476,7 +480,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}},
                                           end}
                     end, {[], Subscriptions}, TopicFilters),
     suback(FromPid, PacketId, ReasonCodes),
-    noreply(State#state{subscriptions = Subscriptions1});
+    noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1}));
 
 %% UNSUBSCRIBE:
 handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}},
@@ -485,7 +489,7 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}},
         lists:foldr(fun({Topic, _SubOpts}, {Acc, SubMap}) ->
                             case maps:find(Topic, SubMap) of
                                 {ok, SubOpts} ->
-                                    ok = emqx_broker:unsubscribe(Topic, ClientId),
+                                    ok = emqx_broker:unsubscribe(Topic),
                                     emqx_hooks:run('session.unsubscribed', [#{client_id => ClientId}, Topic, SubOpts]),
                                     {[?RC_SUCCESS|Acc], maps:remove(Topic, SubMap)};
                                 error ->
@@ -493,36 +497,38 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}},
                             end
                     end, {[], Subscriptions}, TopicFilters),
     unsuback(From, PacketId, ReasonCodes),
-    noreply(State#state{subscriptions = Subscriptions1});
+    noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1}));
 
 %% PUBACK:
 handle_cast({puback, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) ->
+    noreply(
       case emqx_inflight:contain(PacketId, Inflight) of
           true ->
-            noreply(dequeue(acked(puback, PacketId, State)));
+              ensure_stats_timer(dequeue(acked(puback, PacketId, State)));
           false ->
               ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State),
               emqx_metrics:trans(inc, 'packets/puback/missed'),
-            {noreply, State}
+              State
-    end;
+      end);
 
 %% PUBCOMP:
 handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) ->
+    noreply(
       case emqx_inflight:contain(PacketId, Inflight) of
           true ->
-            noreply(dequeue(acked(pubcomp, PacketId, State)));
+              ensure_stats_timer(dequeue(acked(pubcomp, PacketId, State)));
           false ->
               ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State),
               emqx_metrics:trans(inc, 'packets/pubcomp/missed'),
-            {noreply, State}
+              State
-    end;
+      end);
 
 %% RESUME:
 handle_cast({resume, #{conn_pid        := ConnPid,
                        will_msg        := WillMsg,
-                       expiry_interval     := SessionExpiryInterval,
+                       expiry_interval := ExpiryInterval,
-                       max_inflight        := MaxInflight,
+                       max_inflight    := MaxInflight}},
-                       topic_alias_maximum := TopicAliasMaximum}}, State = #state{client_id        = ClientId,
+            State = #state{client_id        = ClientId,
                            conn_pid         = OldConnPid,
                            clean_start      = CleanStart,
                            retry_timer      = RetryTimer,
@@ -533,7 +539,8 @@ handle_cast({resume, #{conn_pid            := ConnPid,
     ?LOG(info, "Resumed by connection ~p ", [ConnPid], State),
 
     %% Cancel Timers
-    lists:foreach(fun emqx_misc:cancel_timer/1, [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]),
+    lists:foreach(fun emqx_misc:cancel_timer/1,
+                  [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]),
 
     case kick(ClientId, OldConnPid, ConnPid) of
         ok -> ?LOG(warning, "Connection ~p kickout ~p", [ConnPid, OldConnPid], State);
@@ -550,9 +557,8 @@ handle_cast({resume, #{conn_pid            := ConnPid,
                          awaiting_rel     = #{},
                          await_rel_timer  = undefined,
                          expiry_timer     = undefined,
-                         expiry_interval     = SessionExpiryInterval,
+                         expiry_interval  = ExpiryInterval,
                          inflight         = emqx_inflight:update_size(MaxInflight, State#state.inflight),
-                         topic_alias_maximum = TopicAliasMaximum,
                          will_delay_timer = undefined,
                          will_msg         = WillMsg},
 
@@ -562,9 +568,9 @@ handle_cast({resume, #{conn_pid            := ConnPid,
     emqx_hooks:run('session.resumed', [#{client_id => ClientId}, attrs(State)]),
 
     %% Replay delivery and Dequeue pending messages
-    noreply(dequeue(retry_delivery(true, State1)));
+    noreply(ensure_stats_timer(dequeue(retry_delivery(true, State1))));
 
-handle_cast({expiry_interval, Interval}, State) ->
+handle_cast({update_expiry_interval, Interval}, State) ->
     {noreply, State#state{expiry_interval = Interval}};
 
 handle_cast(Msg, State) ->
@@ -573,9 +579,10 @@ handle_cast(Msg, State) ->
 
 %% Batch dispatch
 handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) ->
-    {noreply, lists:foldl(fun(Msg, NewState) ->
+    noreply(lists:foldl(
-                                  element(2, handle_info({dispatch, Topic, Msg}, NewState))
+              fun(Msg, St) ->
-                          end, State, Msgs)};
+                  element(2, handle_info({dispatch, Topic, Msg}, St))
+              end, State, Msgs));
 
 %% Dispatch message
 handle_info({dispatch, Topic, Msg = #message{}}, State) ->
@@ -584,12 +591,11 @@ handle_info({dispatch, Topic, Msg = #message{}}, State) ->
             %% Require ack, but we do not have connection
             %% negative ack the message so it can try the next subscriber in the group
             ok = emqx_shared_sub:nack_no_connection(Msg),
-            noreply(State);
+            {noreply, State};
         false ->
-            handle_dispatch(Topic, Msg, State)
+            noreply(ensure_stats_timer(handle_dispatch(Topic, Msg, State)))
     end;
 
-
 %% Do nothing if the client has been disconnected.
 handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) ->
     noreply(State#state{retry_timer = undefined});
@@ -598,11 +604,13 @@ handle_info({timeout, Timer, retry_delivery}, State = #state{retry_timer = Timer
     noreply(retry_delivery(false, State#state{retry_timer = undefined}));
 
 handle_info({timeout, Timer, check_awaiting_rel}, State = #state{await_rel_timer = Timer}) ->
-    noreply(expire_awaiting_rel(State#state{await_rel_timer = undefined}));
+    State1 = State#state{await_rel_timer = undefined},
+    noreply(ensure_stats_timer(expire_awaiting_rel(State1)));
 
 handle_info({timeout, Timer, emit_stats},
             State = #state{client_id = ClientId,
-                           stats_timer = Timer}) ->
+                           stats_timer = Timer,
+                           gc_state = GcState}) ->
     emqx_metrics:commit(),
     _ = emqx_sm:set_session_stats(ClientId, stats(State)),
     NewState = State#state{stats_timer = undefined},
@@ -611,20 +619,27 @@ handle_info({timeout, Timer, emit_stats},
         continue ->
             {noreply, NewState};
         hibernate ->
-            ok = emqx_gc:reset(), %% going to hibernate, reset gc stats
+            %% going to hibernate, reset gc stats
-            {noreply, NewState, hibernate};
+            GcState1 = emqx_gc:reset(GcState),
+            {noreply, NewState#state{gc_state = GcState1}, hibernate};
         {shutdown, Reason} ->
             ?LOG(warning, "shutdown due to ~p", [Reason], NewState),
             shutdown(Reason, NewState)
     end;
+
 handle_info({timeout, Timer, expired}, State = #state{expiry_timer = Timer}) ->
-    ?LOG(info, "expired, shutdown now:(", [], State),
+    ?LOG(info, "expired, shutdown now.", [], State),
     shutdown(expired, State);
 
 handle_info({timeout, Timer, will_delay}, State = #state{will_msg = WillMsg, will_delay_timer = Timer}) ->
     send_willmsg(WillMsg),
     {noreply, State#state{will_msg = undefined}};
 
+%% ConnPid is shutting down by the supervisor.
+handle_info({'EXIT', ConnPid, Reason}, #state{conn_pid = ConnPid})
+    when Reason =:= killed; Reason =:= shutdown ->
+    exit(Reason);
+
 handle_info({'EXIT', ConnPid, Reason}, State = #state{will_msg = WillMsg, expiry_interval = 0, conn_pid = ConnPid}) ->
     send_willmsg(WillMsg),
     {stop, Reason, State#state{will_msg = undefined, conn_pid = undefined}};
@@ -641,47 +656,44 @@ handle_info({'EXIT', Pid, Reason}, State = #state{conn_pid = ConnPid}) ->
     ?LOG(error, "Unexpected EXIT: conn_pid=~p, exit_pid=~p, reason=~p",
          [ConnPid, Pid, Reason], State),
     {noreply, State};
+
 handle_info(Info, State) ->
     emqx_logger:error("[Session] unexpected info: ~p", [Info]),
     {noreply, State}.
 
-terminate(Reason, #state{will_msg = WillMsg, client_id = ClientId, conn_pid = ConnPid}) ->
+terminate(Reason, #state{will_msg = WillMsg,
-    emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]),
+                         client_id = ClientId,
+                         conn_pid = ConnPid,
+                         old_conn_pid = OldConnPid}) ->
     send_willmsg(WillMsg),
-    %% Ensure to shutdown the connection
+    [maybe_shutdown(Pid, Reason) || Pid <- [ConnPid, OldConnPid]],
-    if
+    emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]).
-        ConnPid =/= undefined ->
-            ConnPid ! {shutdown, Reason};
-        true -> ok
-    end,
-    emqx_sm:unregister_session(ClientId).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+maybe_shutdown(undefined, _Reason) ->
+    ok;
+maybe_shutdown(Pid, normal) ->
+     Pid ! {shutdown, normal};
+maybe_shutdown(Pid, Reason) ->
+    exit(Pid, Reason).
+
 %%------------------------------------------------------------------------------
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-has_connection(#state{conn_pid = Pid}) -> is_pid(Pid) andalso is_process_alive(Pid).
+has_connection(#state{conn_pid = Pid}) ->
+    is_pid(Pid) andalso is_process_alive(Pid).
 
-handle_dispatch(Topic, Msg = #message{headers = Headers},
+handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) ->
-                State = #state{subscriptions = SubMap,
+    case maps:find(Topic, SubMap) of
-                               topic_alias_maximum = TopicAliasMaximum
-                              }) ->
-    TopicAlias = maps:get('Topic-Alias', Headers, undefined),
-    if
-        TopicAlias =:= undefined orelse TopicAlias =< TopicAliasMaximum ->
-            noreply(case maps:find(Topic, SubMap) of
         {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} ->
             run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State);
         {ok, #{nl := Nl, qos := QoS, rap := Rap}} ->
             run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State);
         error ->
             dispatch(emqx_message:unset_flag(dup, Msg), State)
-                    end);
-        true ->
-            noreply(State)
     end.
 
 suback(_From, undefined, _ReasonCodes) ->
@@ -925,8 +937,7 @@ dequeue(State = #state{inflight = Inflight}) ->
 
 dequeue2(State = #state{mqueue = Q}) ->
     case emqx_mqueue:out(Q) of
-        {empty, _Q} ->
+        {empty, _Q} -> State;
-            State;
         {{value, Msg}, Q1} ->
             %% Dequeue more
             dequeue(dispatch(Msg, State#state{mqueue = Q1}))
@@ -967,7 +978,8 @@ ensure_will_delay_timer(State = #state{will_msg = WillMsg}) ->
     send_willmsg(WillMsg),
     State#state{will_msg = undefined}.
 
-ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined,
+ensure_stats_timer(State = #state{enable_stats = true,
+                                  stats_timer = undefined,
                                   idle_timeout = IdleTimeout}) ->
     State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)};
 ensure_stats_timer(State) ->
@@ -986,9 +998,8 @@ next_pkt_id(State = #state{next_pkt_id = Id}) ->
 %% Inc stats
 
 inc_stats(deliver, Msg, State = #state{deliver_stats = I}) ->
-    MsgSize = msg_size(Msg),
+    State1 = maybe_gc({1, msg_size(Msg)}, State),
-    ok = emqx_gc:inc(1, MsgSize),
+    State1#state{deliver_stats = I + 1};
-    State#state{deliver_stats = I + 1};
 inc_stats(enqueue, _Msg, State = #state{enqueue_stats = I}) ->
     State#state{enqueue_stats = I + 1}.
 
@@ -1005,10 +1016,17 @@ reply({Reply, State}) ->
     reply(Reply, State).
 
 reply(Reply, State) ->
-    {reply, Reply, ensure_stats_timer(State)}.
+    {reply, Reply, State}.
 
 noreply(State) ->
-    {noreply, ensure_stats_timer(State)}.
+    {noreply, State}.
 
 shutdown(Reason, State) ->
-    {stop, {shutdown, Reason}, State}.
+    {stop, Reason, State}.
+
+maybe_gc(_, State = #state{gc_state = undefined}) ->
+    State;
+maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) ->
+    {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt),
+    State#state{gc_state = GCSt1}.
+

src/emqx_session_sup.erl

@@ -14,31 +14,243 @@
 
 -module(emqx_session_sup).
 
--behavior(supervisor).
+-behaviour(gen_server).
 
--include("emqx.hrl").
+-export([start_link/1]).
+-export([start_session/1, count_sessions/0]).
 
--export([start_link/0, start_session/1]).
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
+         code_change/3]).
 
--export([init/1]).
+-type(shutdown() :: brutal_kill | infinity | pos_integer()).
 
-start_link() ->
+-record(state, {
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+          sessions :: #{pid() => emqx_types:client_id()},
+          mfargs :: mfa(),
+          shutdown :: shutdown(),
+          clean_down :: fun()
+         }).
 
--spec(start_session(map()) -> {ok, pid()}).
+-define(SUP, ?MODULE).
-start_session(Attrs) ->
+-define(BATCH_EXIT, 100000).
-    supervisor:start_child(?MODULE, [Attrs]).
+-define(ERROR_MSG(Format, Args),
+        error_logger:error_msg("[~s] " ++ Format, [?MODULE | Args])).
 
-%%--------------------------------------------------------------------
+%% @doc Start session supervisor.
-%% Supervisor callbacks
+-spec(start_link(map()) -> emqx_types:startlink_ret()).
-%%--------------------------------------------------------------------
+start_link(SessSpec) when is_map(SessSpec) ->
+    gen_server:start_link({local, ?SUP}, ?MODULE, [SessSpec], []).
 
-init([]) ->
+%%------------------------------------------------------------------------------
-    {ok, {{simple_one_for_one, 0, 1},
+%% API
-          [#{id       => session,
+%%------------------------------------------------------------------------------
-             start    => {emqx_session, start_link, []},
+
-             restart  => temporary,
+%% @doc Start a session.
-             shutdown => 5000,
+-spec(start_session(map()) -> emqx_types:startlink_ret()).
-             type     => worker,
+start_session(SessAttrs) ->
-             modules  => [emqx_session]}]}}.
+    gen_server:call(?SUP, {start_session, SessAttrs}, infinity).
+
+%% @doc Count sessions.
+-spec(count_sessions() -> non_neg_integer()).
+count_sessions() ->
+    gen_server:call(?SUP, count_sessions, infinity).
+
+%%------------------------------------------------------------------------------
+%% gen_server callbacks
+%%------------------------------------------------------------------------------
+
+init([Spec]) ->
+    process_flag(trap_exit, true),
+    MFA = maps:get(start, Spec),
+    Shutdown = maps:get(shutdown, Spec, brutal_kill),
+    CleanDown = maps:get(clean_down, Spec, undefined),
+    State = #state{sessions = #{},
+                   mfargs = MFA,
+                   shutdown = Shutdown,
+                   clean_down = CleanDown
+                  },
+    {ok, State}.
+
+handle_call({start_session, SessAttrs = #{client_id := ClientId}}, _From,
+            State = #state{sessions = SessMap, mfargs = {M, F, Args}}) ->
+    try erlang:apply(M, F, [SessAttrs | Args]) of
+        {ok, Pid} ->
+            reply({ok, Pid}, State#state{sessions = maps:put(Pid, ClientId, SessMap)});
+        ignore ->
+            reply(ignore, State);
+        {error, Reason} ->
+            reply({error, Reason}, State)
+    catch
+        _:Error:Stk ->
+            ?ERROR_MSG("Failed to start session ~p: ~p, stacktrace:~n~p",
+                       [ClientId, Error, Stk]),
+            reply({error, Error}, State)
+    end;
+
+handle_call(count_sessions, _From, State = #state{sessions = SessMap}) ->
+    {reply, maps:size(SessMap), State};
+
+handle_call(Req, _From, State) ->
+    ?ERROR_MSG("unexpected call: ~p", [Req]),
+    {reply, ignored, State}.
+
+handle_cast(Msg, State) ->
+    ?ERROR_MSG("unexpected cast: ~p", [Msg]),
+    {noreply, State}.
+
+handle_info({'EXIT', Pid, _Reason}, State = #state{sessions = SessMap, clean_down = CleanDown}) ->
+    SessPids = [Pid | drain_exit(?BATCH_EXIT, [])],
+    {SessItems, SessMap1} = erase_all(SessPids, SessMap),
+    (CleanDown =:= undefined)
+        orelse emqx_pool:async_submit(
+                 fun lists:foreach/2, [CleanDown, SessItems]),
+    {noreply, State#state{sessions = SessMap1}};
+
+handle_info(Info, State) ->
+    ?ERROR_MSG("unexpected info: ~p", [Info]),
+    {noreply, State}.
+
+terminate(_Reason, State) ->
+    terminate_children(State).
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+drain_exit(0, Acc) ->
+    lists:reverse(Acc);
+drain_exit(Cnt, Acc) ->
+    receive
+        {'EXIT', Pid, _Reason} ->
+            drain_exit(Cnt - 1, [Pid|Acc])
+    after 0 ->
+          lists:reverse(Acc)
+    end.
+
+erase_all(Pids, Map) ->
+    lists:foldl(
+      fun(Pid, {Acc, M}) ->
+            case maps:take(Pid, M) of
+                {Val, M1} ->
+                    {[{Val, Pid}|Acc], M1};
+                error ->
+                    {Acc, M}
+            end
+      end, {[], Map}, Pids).
+
+terminate_children(State = #state{sessions = SessMap, shutdown = Shutdown}) ->
+    {Pids, EStack0} = monitor_children(SessMap),
+    Sz = sets:size(Pids),
+    EStack =
+    case Shutdown of
+        brutal_kill ->
+            sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids),
+            wait_children(Shutdown, Pids, Sz, undefined, EStack0);
+        infinity ->
+            sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids),
+            wait_children(Shutdown, Pids, Sz, undefined, EStack0);
+        Time when is_integer(Time) ->
+            sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids),
+            TRef = erlang:start_timer(Time, self(), kill),
+            wait_children(Shutdown, Pids, Sz, TRef, EStack0)
+    end,
+    %% Unroll stacked errors and report them
+    dict:fold(fun(Reason, Pid, _) ->
+                  report_error(connection_shutdown_error, Reason, Pid, State)
+              end, ok, EStack).
+
+monitor_children(SessMap) ->
+    lists:foldl(
+      fun(Pid, {Pids, EStack}) ->
+          case monitor_child(Pid) of
+              ok ->
+                  {sets:add_element(Pid, Pids), EStack};
+              {error, normal} ->
+                  {Pids, EStack};
+              {error, Reason} ->
+                  {Pids, dict:append(Reason, Pid, EStack)}
+          end
+      end, {sets:new(), dict:new()}, maps:keys(SessMap)).
+
+%% Help function to shutdown/2 switches from link to monitor approach
+monitor_child(Pid) ->
+    %% Do the monitor operation first so that if the child dies
+    %% before the monitoring is done causing a 'DOWN'-message with
+    %% reason noproc, we will get the real reason in the 'EXIT'-message
+    %% unless a naughty child has already done unlink...
+    erlang:monitor(process, Pid),
+    unlink(Pid),
+
+    receive
+	%% If the child dies before the unlik we must empty
+	%% the mail-box of the 'EXIT'-message and the 'DOWN'-message.
+	{'EXIT', Pid, Reason} ->
+	    receive
+		{'DOWN', _, process, Pid, _} ->
+		    {error, Reason}
+	    end
+    after 0 ->
+	    %% If a naughty child did unlink and the child dies before
+	    %% monitor the result will be that shutdown/2 receives a
+	    %% 'DOWN'-message with reason noproc.
+	    %% If the child should die after the unlink there
+	    %% will be a 'DOWN'-message with a correct reason
+	    %% that will be handled in shutdown/2.
+	    ok
+    end.
+
+wait_children(_Shutdown, _Pids, 0, undefined, EStack) ->
+    EStack;
+wait_children(_Shutdown, _Pids, 0, TRef, EStack) ->
+	%% If the timer has expired before its cancellation, we must empty the
+	%% mail-box of the 'timeout'-message.
+    erlang:cancel_timer(TRef),
+    receive
+        {timeout, TRef, kill} ->
+            EStack
+    after 0 ->
+            EStack
+    end;
+
+%%TODO: Copied from supervisor.erl, rewrite it later.
+wait_children(brutal_kill, Pids, Sz, TRef, EStack) ->
+    receive
+        {'DOWN', _MRef, process, Pid, killed} ->
+            wait_children(brutal_kill, sets:del_element(Pid, Pids), Sz-1, TRef, EStack);
+
+        {'DOWN', _MRef, process, Pid, Reason} ->
+            wait_children(brutal_kill, sets:del_element(Pid, Pids),
+                          Sz-1, TRef, dict:append(Reason, Pid, EStack))
+    end;
+
+wait_children(Shutdown, Pids, Sz, TRef, EStack) ->
+    receive
+        {'DOWN', _MRef, process, Pid, shutdown} ->
+            wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack);
+        {'DOWN', _MRef, process, Pid, normal} ->
+            wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack);
+        {'DOWN', _MRef, process, Pid, Reason} ->
+            wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1,
+                          TRef, dict:append(Reason, Pid, EStack));
+        {timeout, TRef, kill} ->
+            sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids),
+            wait_children(Shutdown, Pids, Sz-1, undefined, EStack)
+    end.
+
+report_error(Error, Reason, Pid, #state{mfargs = MFA}) ->
+    SupName  = list_to_atom("esockd_connection_sup - " ++ pid_to_list(self())),
+    ErrorMsg = [{supervisor, SupName},
+                {errorContext, Error},
+                {reason, Reason},
+                {offender, [{pid, Pid},
+                            {name, connection},
+                            {mfargs, MFA}]}],
+    error_logger:error_report(supervisor_report, ErrorMsg).
+
+reply(Repy, State) ->
+    {reply, Repy, State}.
 

src/emqx_shared_sub.erl

@@ -17,6 +17,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_mqtt.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
@@ -27,7 +28,8 @@
 -export([start_link/0]).
 
 -export([subscribe/3, unsubscribe/3]).
--export([dispatch/3, maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]).
+-export([dispatch/3]).
+-export([maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]).
 
 %% for testing
 -export([subscribers/2]).
@@ -38,6 +40,7 @@
 
 -define(SERVER, ?MODULE).
 -define(TAB, emqx_shared_subscription).
+-define(SHARED_SUBS, emqx_shared_subscriber).
 -define(ALIVE_SUBS, emqx_alive_shared_subscribers).
 -define(SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS, 5).
 -define(ack, shared_sub_ack).
@@ -48,8 +51,6 @@
 -record(state, {pmon}).
 -record(emqx_shared_subscription, {group, topic, subpid}).
 
--include("emqx_mqtt.hrl").
-
 %%------------------------------------------------------------------------------
 %% Mnesia bootstrap
 %%------------------------------------------------------------------------------
@@ -72,16 +73,11 @@ mnesia(copy) ->
 start_link() ->
     gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
 
-subscribe(undefined, _Topic, _SubPid) ->
-    ok;
 subscribe(Group, Topic, SubPid) when is_pid(SubPid) ->
-    mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)),
+    gen_server:call(?SERVER, {subscribe, Group, Topic, SubPid}).
-    gen_server:cast(?SERVER, {monitor, SubPid}).
 
-unsubscribe(undefined, _Topic, _SubPid) ->
-    ok;
 unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) ->
-    mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)).
+    gen_server:call(?SERVER, {unsubscribe, Group, Topic, SubPid}).
 
 record(Group, Topic, SubPid) ->
     #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}.
@@ -251,14 +247,15 @@ do_pick_subscriber(Group, Topic, round_robin, _ClientId, Count) ->
 subscribers(Group, Topic) ->
     ets:select(?TAB, [{{emqx_shared_subscription, Group, Topic, '$1'}, [], ['$1']}]).
 
-%%-----------------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% gen_server callbacks
-%%-----------------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 init([]) ->
-    {atomic, PMon} = mnesia:transaction(fun init_monitors/0),
     mnesia:subscribe({table, ?TAB, simple}),
-    ets:new(?ALIVE_SUBS, [named_table, {read_concurrency, true}, protected]),
+    {atomic, PMon} = mnesia:transaction(fun init_monitors/0),
+    ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]),
+    ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]),
     {ok, update_stats(#state{pmon = PMon})}.
 
 init_monitors() ->
@@ -267,14 +264,29 @@ init_monitors() ->
           emqx_pmon:monitor(SubPid, Mon)
       end, emqx_pmon:new(), ?TAB).
 
+handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) ->
+    mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)),
+    case ets:member(?SHARED_SUBS, {Group, Topic}) of
+        true  -> ok;
+        false -> ok = emqx_router:do_add_route(Topic, {Group, node()})
+    end,
+    ok = maybe_insert_alive_tab(SubPid),
+    true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}),
+    {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})};
+
+handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) ->
+    mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)),
+    true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}),
+    case ets:member(?SHARED_SUBS, {Group, Topic}) of
+        true  -> ok;
+        false -> ok = emqx_router:do_delete_route(Topic, {Group, node()})
+    end,
+    {reply, ok, State};
+
 handle_call(Req, _From, State) ->
     emqx_logger:error("[SharedSub] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({monitor, SubPid}, State= #state{pmon = PMon}) ->
-    NewPmon = emqx_pmon:monitor(SubPid, PMon),
-    ok = maybe_insert_alive_tab(SubPid),
-    {noreply, update_stats(State#state{pmon = NewPmon})};
 handle_cast(Msg, State) ->
     emqx_logger:error("[SharedSub] unexpected cast: ~p", [Msg]),
     {noreply, State}.
@@ -316,12 +328,18 @@ maybe_insert_alive_tab(Pid) when is_pid(Pid) -> ets:insert(?ALIVE_SUBS, {Pid}),
 cleanup_down(SubPid) ->
     ?IS_LOCAL_PID(SubPid) orelse ets:delete(?ALIVE_SUBS, SubPid),
     lists:foreach(
-        fun(Record) ->
+        fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) ->
-            mnesia:dirty_delete_object(?TAB, Record)
+            ok = mnesia:dirty_delete_object(?TAB, Record),
+            true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}),
+            case ets:member(?SHARED_SUBS, {Group, Topic}) of
+                true -> ok;
+                false -> ok = emqx_router:do_delete_route(Topic, {Group, node()})
+            end
         end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})).
 
 update_stats(State) ->
-    emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), State.
+    emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)),
+    State.
 
 %% Return 'true' if the subscriber process is alive AND not in the failed list
 is_active_sub(Pid, FailedSubs) ->

src/emqx_sm.erl

@@ -21,12 +21,15 @@
 -export([start_link/0]).
 
 -export([open_session/1, close_session/1]).
--export([lookup_session/1, lookup_session_pid/1]).
 -export([resume_session/2]).
 -export([discard_session/1, discard_session/2]).
--export([register_session/2, unregister_session/1]).
+-export([register_session/1, register_session/2]).
--export([get_session_attrs/1, set_session_attrs/2]).
+-export([unregister_session/1, unregister_session/2]).
--export([get_session_stats/1, set_session_stats/2]).
+-export([get_session_attrs/1, get_session_attrs/2,
+         set_session_attrs/2, set_session_attrs/3]).
+-export([get_session_stats/1, get_session_stats/2,
+         set_session_stats/2, set_session_stats/3]).
+-export([lookup_session_pids/1]).
 
 %% Internal functions for rpc
 -export([dispatch/3]).
@@ -34,18 +37,23 @@
 %% Internal function for stats
 -export([stats_fun/0]).
 
+%% Internal function for emqx_session_sup
+-export([clean_down/1]).
+
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
 -define(SM, ?MODULE).
 
-%% ETS Tables
+%% ETS Tables for session management.
 -define(SESSION_TAB, emqx_session).
--define(SESSION_P_TAB,     emqx_persistent_session).
+-define(SESSION_P_TAB, emqx_session_p).
 -define(SESSION_ATTRS_TAB, emqx_session_attrs).
 -define(SESSION_STATS_TAB, emqx_session_stats).
 
+-define(BATCH_SIZE, 100000).
+
 -spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
     gen_server:start_link({local, ?SM}, ?MODULE, [], []).
@@ -59,12 +67,11 @@ open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid
                  end,
     emqx_sm_locker:trans(ClientId, CleanStart);
 
-open_session(SessAttrs = #{clean_start := false,
+open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) ->
-                           client_id   := ClientId}) ->
     ResumeStart = fun(_) ->
                       case resume_session(ClientId, SessAttrs) of
-                          {ok, SPid} ->
+                          {ok, SessPid} ->
-                              {ok, SPid, true};
+                              {ok, SessPid, true};
                           {error, not_found} ->
                               emqx_session_sup:start_session(SessAttrs)
                       end
@@ -76,168 +83,162 @@ open_session(SessAttrs = #{clean_start := false,
 discard_session(ClientId) when is_binary(ClientId) ->
     discard_session(ClientId, self()).
 
+-spec(discard_session(emqx_types:client_id(), pid()) -> ok).
 discard_session(ClientId, ConnPid) when is_binary(ClientId) ->
-    lists:foreach(fun({_ClientId, SPid}) ->
+    lists:foreach(
-                      case catch emqx_session:discard(SPid, ConnPid) of
+      fun(SessPid) ->
-                          {Err, Reason} when Err =:= 'EXIT'; Err =:= error ->
+          try emqx_session:discard(SessPid, ConnPid)
-                              emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]);
+          catch
-                          ok -> ok
+              _:Error:_Stk ->
+                  emqx_logger:error("[SM] Failed to discard ~p: ~p", [SessPid, Error])
           end
-                  end, lookup_session(ClientId)).
+      end, lookup_session_pids(ClientId)).
 
 %% @doc Try to resume a session.
 -spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}).
 resume_session(ClientId, SessAttrs = #{conn_pid := ConnPid}) ->
-    case lookup_session(ClientId) of
+    case lookup_session_pids(ClientId) of
         [] -> {error, not_found};
-        [{_ClientId, SPid}] ->
+        [SessPid] ->
-            ok = emqx_session:resume(SPid, SessAttrs),
+            ok = emqx_session:resume(SessPid, SessAttrs),
-            {ok, SPid};
+            {ok, SessPid};
-        Sessions ->
+        SessPids ->
-            [{_, SPid}|StaleSessions] = lists:reverse(Sessions),
+            [SessPid|StalePids] = lists:reverse(SessPids),
-            emqx_logger:error("[SM] More than one session found: ~p", [Sessions]),
+            emqx_logger:error("[SM] More than one session found: ~p", [SessPids]),
-            lists:foreach(fun({_, StalePid}) ->
+            lists:foreach(fun(StalePid) ->
                               catch emqx_session:discard(StalePid, ConnPid)
-                          end, StaleSessions),
+                          end, StalePids),
-            ok = emqx_session:resume(SPid, SessAttrs),
+            ok = emqx_session:resume(SessPid, SessAttrs),
-            {ok, SPid}
+            {ok, SessPid}
     end.
 
 %% @doc Close a session.
--spec(close_session({emqx_types:client_id(), pid()} | pid()) -> ok).
+-spec(close_session(emqx_types:client_id() | pid()) -> ok).
-close_session({_ClientId, SPid}) ->
+close_session(ClientId) when is_binary(ClientId) ->
-    emqx_session:close(SPid);
+    case lookup_session_pids(ClientId) of
-close_session(SPid) when is_pid(SPid) ->
+        [] -> ok;
-    emqx_session:close(SPid).
+        [SessPid] -> close_session(SessPid);
+        SessPids -> lists:foreach(fun close_session/1, SessPids)
+    end;
 
-%% @doc Register a session with attributes.
+close_session(SessPid) when is_pid(SessPid) ->
--spec(register_session(emqx_types:client_id() | {emqx_types:client_id(), pid()},
+    emqx_session:close(SessPid).
-                       list(emqx_session:attr())) -> ok).
-register_session(ClientId, SessAttrs) when is_binary(ClientId) ->
-    register_session({ClientId, self()}, SessAttrs);
 
-register_session(Session = {ClientId, SPid}, SessAttrs)
+%% @doc Register a session.
-    when is_binary(ClientId), is_pid(SPid) ->
+-spec(register_session(emqx_types:client_id()) -> ok).
-    ets:insert(?SESSION_TAB, Session),
+register_session(ClientId) when is_binary(ClientId) ->
-    ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}),
+    register_session(ClientId, self()).
-    proplists:get_value(clean_start, SessAttrs, true)
-        andalso ets:insert(?SESSION_P_TAB, Session),
-    emqx_sm_registry:register_session(Session),
-    notify({registered, ClientId, SPid}).
 
-%% @doc Get session attrs
+-spec(register_session(emqx_types:client_id(), pid()) -> ok).
--spec(get_session_attrs({emqx_types:client_id(), pid()}) -> list(emqx_session:attr())).
+register_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) ->
-get_session_attrs(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) ->
+    Session = {ClientId, SessPid},
-    safe_lookup_element(?SESSION_ATTRS_TAB, Session, []).
+    true = ets:insert(?SESSION_TAB, Session),
-
+    emqx_sm_registry:register_session(Session).
-%% @doc Set session attrs
--spec(set_session_attrs(emqx_types:client_id() | {emqx_types:client_id(), pid()},
-                        list(emqx_session:attr())) -> true).
-set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) ->
-    set_session_attrs({ClientId, self()}, SessAttrs);
-set_session_attrs(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) ->
-    ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}).
 
 %% @doc Unregister a session
--spec(unregister_session(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok).
+-spec(unregister_session(emqx_types:client_id()) -> ok).
 unregister_session(ClientId) when is_binary(ClientId) ->
-    unregister_session({ClientId, self()});
+    unregister_session(ClientId, self()).
 
-unregister_session(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) ->
+-spec(unregister_session(emqx_types:client_id(), pid()) -> ok).
-    emqx_sm_registry:unregister_session(Session),
+unregister_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) ->
-    ets:delete(?SESSION_STATS_TAB, Session),
+    Session = {ClientId, SessPid},
-    ets:delete(?SESSION_ATTRS_TAB, Session),
+    true = ets:delete(?SESSION_STATS_TAB, Session),
-    ets:delete_object(?SESSION_P_TAB, Session),
+    true = ets:delete(?SESSION_ATTRS_TAB, Session),
-    ets:delete_object(?SESSION_TAB, Session),
+    true = ets:delete_object(?SESSION_P_TAB, Session),
-    notify({unregistered, ClientId, SPid}).
+    true = ets:delete_object(?SESSION_TAB, Session),
+    emqx_sm_registry:unregister_session(Session).
+
+%% @doc Get session attrs
+-spec(get_session_attrs(emqx_types:client_id()) -> list(emqx_session:attr())).
+get_session_attrs(ClientId) when is_binary(ClientId) ->
+    case lookup_session_pids(ClientId) of
+        [] -> [];
+        [SessPid|_] -> get_session_attrs(ClientId, SessPid)
+    end.
+
+-spec(get_session_attrs(emqx_types:client_id(), pid()) -> list(emqx_session:attr())).
+get_session_attrs(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) ->
+    emqx_tables:lookup_value(?SESSION_ATTRS_TAB, {ClientId, SessPid}, []).
+
+%% @doc Set session attrs
+-spec(set_session_attrs(emqx_types:client_id(), list(emqx_session:attr())) -> true).
+set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) ->
+    set_session_attrs(ClientId, self(), SessAttrs).
+
+-spec(set_session_attrs(emqx_types:client_id(), pid(), list(emqx_session:attr())) -> true).
+set_session_attrs(ClientId, SessPid, SessAttrs) when is_binary(ClientId), is_pid(SessPid) ->
+    Session = {ClientId, SessPid},
+    true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}),
+    proplists:get_value(clean_start, SessAttrs, true) orelse ets:insert(?SESSION_P_TAB, Session).
 
 %% @doc Get session stats
--spec(get_session_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())).
+-spec(get_session_stats(emqx_types:client_id()) -> list(emqx_stats:stats())).
-get_session_stats(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) ->
+get_session_stats(ClientId) when is_binary(ClientId) ->
-    safe_lookup_element(?SESSION_STATS_TAB, Session, []).
+    case lookup_session_pids(ClientId) of
+        [] -> [];
+        [SessPid|_] ->
+            get_session_stats(ClientId, SessPid)
+    end.
+
+-spec(get_session_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())).
+get_session_stats(ClientId, SessPid) when is_binary(ClientId) ->
+    emqx_tables:lookup_value(?SESSION_STATS_TAB, {ClientId, SessPid}, []).
 
 %% @doc Set session stats
--spec(set_session_stats(emqx_types:client_id() | {emqx_types:client_id(), pid()},
+-spec(set_session_stats(emqx_types:client_id(), emqx_stats:stats()) -> true).
-                        emqx_stats:stats()) -> true).
 set_session_stats(ClientId, Stats) when is_binary(ClientId) ->
-    set_session_stats({ClientId, self()}, Stats);
+    set_session_stats(ClientId, self(), Stats).
-set_session_stats(Session = {ClientId, SPid}, Stats) when is_binary(ClientId), is_pid(SPid) ->
-    ets:insert(?SESSION_STATS_TAB, {Session, Stats}).
 
-%% @doc Lookup a session from registry
+-spec(set_session_stats(emqx_types:client_id(), pid(), emqx_stats:stats()) -> true).
--spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})).
+set_session_stats(ClientId, SessPid, Stats) when is_binary(ClientId), is_pid(SessPid) ->
-lookup_session(ClientId) ->
+    ets:insert(?SESSION_STATS_TAB, {{ClientId, SessPid}, Stats}).
+
+%% @doc Lookup session pid.
+-spec(lookup_session_pids(emqx_types:client_id()) -> list(pid())).
+lookup_session_pids(ClientId) ->
     case emqx_sm_registry:is_enabled() of
         true -> emqx_sm_registry:lookup_session(ClientId);
-        false -> ets:lookup(?SESSION_TAB, ClientId)
+        false -> emqx_tables:lookup_value(?SESSION_TAB, ClientId, [])
     end.
 
 %% @doc Dispatch a message to the session.
 -spec(dispatch(emqx_types:client_id(), emqx_topic:topic(), emqx_types:message()) -> any()).
 dispatch(ClientId, Topic, Msg) ->
-    case lookup_session_pid(ClientId) of
+    case lookup_session_pids(ClientId) of
-        Pid when is_pid(Pid) ->
+        [SessPid|_] when is_pid(SessPid) ->
-            Pid ! {dispatch, Topic, Msg};
+            SessPid ! {dispatch, Topic, Msg};
-        undefined ->
+        [] ->
             emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg])
     end.
 
-%% @doc Lookup session pid.
--spec(lookup_session_pid(emqx_types:client_id()) -> pid() | undefined).
-lookup_session_pid(ClientId) ->
-    safe_lookup_element(?SESSION_TAB, ClientId, undefined).
-
-safe_lookup_element(Tab, Key, Default) ->
-    try ets:lookup_element(Tab, Key, 2)
-    catch
-        error:badarg -> Default
-    end.
-
-notify(Event) ->
-    gen_server:cast(?SM, {notify, Event}).
-
 %%------------------------------------------------------------------------------
 %% gen_server callbacks
 %%------------------------------------------------------------------------------
 
 init([]) ->
     TabOpts = [public, set, {write_concurrency, true}],
-    _ = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]),
+    ok = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]),
-    _ = emqx_tables:new(?SESSION_P_TAB, TabOpts),
+    ok = emqx_tables:new(?SESSION_P_TAB, TabOpts),
-    _ = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts),
+    ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts),
-    _ = emqx_tables:new(?SESSION_STATS_TAB, TabOpts),
+    ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts),
-    emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0),
+    ok = emqx_stats:update_interval(sess_stats, fun ?MODULE:stats_fun/0),
-    {ok, #{session_pmon => emqx_pmon:new()}}.
+    {ok, #{}}.
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[SM] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({notify, {registered, ClientId, SPid}}, State = #{session_pmon := PMon}) ->
-    {noreply, State#{session_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}};
-
-handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{session_pmon := PMon}) ->
-    {noreply, State#{session_pmon := emqx_pmon:demonitor(SPid, PMon)}};
-
 handle_cast(Msg, State) ->
     emqx_logger:error("[SM] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{session_pmon := PMon}) ->
-    case emqx_pmon:find(DownPid, PMon) of
-        undefined ->
-            {noreply, State};
-        ClientId  ->
-            unregister_session({ClientId, DownPid}),
-            {noreply, State#{session_pmon := emqx_pmon:erase(DownPid, PMon)}}
-    end;
-
 handle_info(Info, State) ->
     emqx_logger:error("[SM] unexpected info: ~p", [Info]),
     {noreply, State}.
 
 terminate(_Reason, _State) ->
-    emqx_stats:cancel_update(sm_stats).
+    emqx_stats:cancel_update(sess_stats).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
@@ -246,6 +247,14 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
+clean_down(Session = {ClientId, SessPid}) ->
+    case ets:member(?SESSION_TAB, ClientId)
+         orelse ets:member(?SESSION_ATTRS_TAB, Session) of
+        true ->
+            unregister_session(ClientId, SessPid);
+        false -> ok
+    end.
+
 stats_fun() ->
     safe_update_stats(?SESSION_TAB, 'sessions/count', 'sessions/max'),
     safe_update_stats(?SESSION_P_TAB, 'sessions/persistent/count', 'sessions/persistent/max').

src/emqx_sm_locker.erl

@@ -21,7 +21,7 @@
 -export([trans/2, trans/3]).
 -export([lock/1, lock/2, unlock/1]).
 
--spec(start_link() -> {ok, pid()} | ignore | {error, term()}).
+-spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
     ekka_locker:start_link(?MODULE).
 

src/emqx_sm_registry.erl

@@ -20,7 +20,6 @@
 
 -export([start_link/0]).
 -export([is_enabled/0]).
-
 -export([register_session/1, lookup_session/1, unregister_session/1]).
 
 %% gen_server callbacks
@@ -42,20 +41,25 @@ start_link() ->
 
 -spec(is_enabled() -> boolean()).
 is_enabled() ->
-    ets:info(?TAB, name) =/= undefined.
+    emqx_config:get_env(enable_session_registry, true).
 
--spec(lookup_session(emqx_types:client_id())
+-spec(lookup_session(emqx_types:client_id()) -> list(session_pid())).
-      -> list({emqx_types:client_id(), session_pid()})).
 lookup_session(ClientId) ->
-    [{ClientId, SessPid} || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)].
+    [SessPid || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)].
 
 -spec(register_session({emqx_types:client_id(), session_pid()}) -> ok).
 register_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) ->
-    mnesia:dirty_write(?TAB, record(ClientId, SessPid)).
+    case is_enabled() of
+        true -> mnesia:dirty_write(?TAB, record(ClientId, SessPid));
+        false -> ok
+    end.
 
 -spec(unregister_session({emqx_types:client_id(), session_pid()}) -> ok).
 unregister_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) ->
-    mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)).
+    case is_enabled() of
+        true -> mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid));
+        false -> ok
+    end.
 
 record(ClientId, SessPid) ->
     #global_session{sid = ClientId, pid = SessPid}.
@@ -73,7 +77,7 @@ init([]) ->
                 {storage_properties, [{ets, [{read_concurrency, true},
                                              {write_concurrency, true}]}]}]),
     ok = ekka_mnesia:copy_table(?TAB),
-    _ = ekka:monitor(membership),
+    ok = ekka:monitor(membership),
     {ok, #{}}.
 
 handle_call(Req, _From, State) ->

src/emqx_sm_sup.erl

@@ -1,6 +1,5 @@
 %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
-%%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
 %% You may obtain a copy of the License at
@@ -31,20 +30,35 @@ init([]) ->
                restart => permanent,
                shutdown => 5000,
                type => worker,
-               modules  => [emqx_sm_locker]},
+               modules => [emqx_sm_locker]
+              },
     %% Session registry
     Registry = #{id => registry,
                  start => {emqx_sm_registry, start_link, []},
                  restart => permanent,
                  shutdown => 5000,
                  type => worker,
-                 modules  => [emqx_sm_registry]},
+                 modules => [emqx_sm_registry]
+                },
     %% Session Manager
     Manager = #{id => manager,
                 start => {emqx_sm, start_link, []},
                 restart => permanent,
                 shutdown => 5000,
                 type => worker,
-                modules  => [emqx_sm]},
+                modules => [emqx_sm]
-    {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager]}}.
+               },
+    %% Session Sup
+    SessSpec = #{start => {emqx_session, start_link, []},
+                 shutdown => brutal_kill,
+                 clean_down => fun emqx_sm:clean_down/1
+                },
+    SessionSup = #{id => session_sup,
+                   start => {emqx_session_sup, start_link, [SessSpec ]},
+                   restart => transient,
+                   shutdown => infinity,
+                   type => supervisor,
+                   modules => [emqx_session_sup]
+                  },
+    {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager, SessionSup]}}.
 

src/emqx_stats.erl

@@ -152,7 +152,7 @@ cast(Msg) ->
 %%------------------------------------------------------------------------------
 
 init(#{tick_ms := TickMs}) ->
-    _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]),
+    ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]),
     Stats = lists:append([?CONNECTION_STATS, ?SESSION_STATS, ?PUBSUB_STATS,
                           ?ROUTE_STATS, ?RETAINED_STATS]),
     true = ets:insert(?TAB, [{Name, 0} || Name <- Stats]),

src/emqx_sup.erl

@@ -69,8 +69,6 @@ init([]) ->
     AccessControl = worker_spec(emqx_access_control),
     %% Session Manager
     SMSup = supervisor_spec(emqx_sm_sup),
-    %% Session Sup
-    SessionSup = supervisor_spec(emqx_session_sup),
     %% Connection Manager
     CMSup = supervisor_spec(emqx_cm_sup),
     %% Sys Sup
@@ -83,7 +81,6 @@ init([]) ->
            BridgeSup,
            AccessControl,
            SMSup,
-           SessionSup,
            CMSup,
            SysSup]}}.
 

src/emqx_tables.erl

@@ -15,12 +15,28 @@
 -module(emqx_tables).
 
 -export([new/2]).
+-export([lookup_value/2, lookup_value/3]).
 
 %% Create a named_table ets.
+-spec(new(atom(), list()) -> ok).
 new(Tab, Opts) ->
     case ets:info(Tab, name) of
         undefined ->
-            ets:new(Tab, lists:usort([named_table | Opts]));
+            _ = ets:new(Tab, lists:usort([named_table | Opts])),
-        Tab -> Tab
+            ok;
+        Tab -> ok
+    end.
+
+%% KV lookup
+-spec(lookup_value(atom(), term()) -> any()).
+lookup_value(Tab, Key) ->
+    lookup_value(Tab, Key, undefined).
+
+-spec(lookup_value(atom(), term(), any()) -> any()).
+lookup_value(Tab, Key, Def) ->
+    try
+        ets:lookup_element(Tab, Key, 2)
+    catch
+        error:badarg -> Def
     end.
 

src/emqx_trie.erl

@@ -36,7 +36,7 @@
 %% @doc Create or replicate trie tables.
 -spec(mnesia(boot | copy) -> ok).
 mnesia(boot) ->
-    %% Optimize
+    %% Optimize storage
     StoreProps = [{ets, [{read_concurrency, true},
                          {write_concurrency, true}]}],
     %% Trie table
@@ -72,7 +72,7 @@ insert(Topic) when is_binary(Topic) ->
             write_trie_node(TrieNode#trie_node{topic = Topic});
         [] ->
             %% Add trie path
-            lists:foreach(fun add_path/1, emqx_topic:triples(Topic)),
+            ok = lists:foreach(fun add_path/1, emqx_topic:triples(Topic)),
             %% Add last node
             write_trie_node(#trie_node{node_id = Topic, topic = Topic})
     end.
@@ -93,7 +93,7 @@ lookup(NodeId) ->
 delete(Topic) when is_binary(Topic) ->
     case mnesia:wread({?TRIE_NODE, Topic}) of
         [#trie_node{edge_count = 0}] ->
-            mnesia:delete({?TRIE_NODE, Topic}),
+            ok = mnesia:delete({?TRIE_NODE, Topic}),
             delete_path(lists:reverse(emqx_topic:triples(Topic)));
         [TrieNode] ->
             write_trie_node(TrieNode#trie_node{topic = undefined});
@@ -112,12 +112,12 @@ add_path({Node, Word, Child}) ->
         [TrieNode = #trie_node{edge_count = Count}] ->
             case mnesia:wread({?TRIE, Edge}) of
                 [] ->
-                    write_trie_node(TrieNode#trie_node{edge_count = Count + 1}),
+                    ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}),
                     write_trie(#trie{edge = Edge, node_id = Child});
                 [_] -> ok
             end;
         [] ->
-            write_trie_node(#trie_node{node_id = Node, edge_count = 1}),
+            ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}),
             write_trie(#trie{edge = Edge, node_id = Child})
     end.
 
@@ -154,10 +154,10 @@ match_node(NodeId, [W|Words], ResAcc) ->
 delete_path([]) ->
     ok;
 delete_path([{NodeId, Word, _} | RestPath]) ->
-    mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}),
+    ok = mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}),
-    case mnesia:read(?TRIE_NODE, NodeId) of
+    case mnesia:wread({?TRIE_NODE, NodeId}) of
         [#trie_node{edge_count = 1, topic = undefined}] ->
-            mnesia:delete({?TRIE_NODE, NodeId}),
+            ok = mnesia:delete({?TRIE_NODE, NodeId}),
             delete_path(RestPath);
         [TrieNode = #trie_node{edge_count = 1, topic = _}] ->
             write_trie_node(TrieNode#trie_node{edge_count = 0});
@@ -167,9 +167,11 @@ delete_path([{NodeId, Word, _} | RestPath]) ->
             mnesia:abort({node_not_found, NodeId})
     end.
 
+%% @private
 write_trie(Trie) ->
     mnesia:write(?TRIE, Trie, write).
 
+%% @private
 write_trie_node(TrieNode) ->
     mnesia:write(?TRIE_NODE, TrieNode, write).
 

src/emqx_vm.erl

@@ -1,5 +1,4 @@
-%%--------------------------------------------------------------------
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
-%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -12,7 +11,6 @@
 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
-%%--------------------------------------------------------------------
 
 -module(emqx_vm).
 

src/emqx_zone.erl

@@ -68,7 +68,7 @@ stop() ->
 %%------------------------------------------------------------------------------
 
 init([]) ->
-    _ = emqx_tables:new(?TAB, [set, {read_concurrency, true}]),
+    ok = emqx_tables:new(?TAB, [set, {read_concurrency, true}]),
     {ok, element(2, handle_info(reload, #{timer => undefined}))}.
 
 handle_call(force_reload, _From, State) ->

test/emqx_banned_SUITE.erl

@@ -18,9 +18,7 @@
 -compile(nowarn_export_all).
 
 -include("emqx.hrl").
-
 -include("emqx_mqtt.hrl").
-
 -include_lib("eunit/include/eunit.hrl").
 
 all() -> [t_banned_all].
@@ -36,11 +34,20 @@ t_banned_all(_) ->
                      until = TimeNow + 1},
     ok = emqx_banned:add(Banned),
     % here is not expire banned test because its check interval is greater than 5 mins, but its effect has been confirmed
-    ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})),
+    ?assert(emqx_banned:check(#{client_id => <<"TestClient">>,
+                                username => undefined,
+                                peername => {undefined, undefined}})),
     timer:sleep(2500),
-    ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})),
+    ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>,
+                                   username => undefined,
+                                   peername => {undefined, undefined}})),
     ok = emqx_banned:add(Banned),
-    ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})),
+    ?assert(emqx_banned:check(#{client_id => <<"TestClient">>,
-    emqx_banned:del({client_id, <<"TestClient">>}),
+                                username => undefined,
-    ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})),
+                                peername => {undefined, undefined}})),
+    emqx_banned:delete({client_id, <<"TestClient">>}),
+    ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>,
+                                   username => undefined,
+                                   peername => {undefined, undefined}})),
     emqx_ct_broker_helpers:run_teardown_steps().
+

test/emqx_broker_SUITE.erl

@@ -36,7 +36,6 @@ groups() ->
     [
      {pubsub, [sequence], [subscribe_unsubscribe,
                            publish, pubsub,
-                           t_local_subscribe,
                            t_shared_subscribe,
                            dispatch_with_no_sub,
                            'pubsub#', 'pubsub+']},
@@ -61,14 +60,14 @@ subscribe_unsubscribe(_) ->
     ok = emqx:subscribe(<<"topic">>, <<"clientId">>),
     ok = emqx:subscribe(<<"topic/1">>, <<"clientId">>, #{ qos => 1 }),
     ok = emqx:subscribe(<<"topic/2">>, <<"clientId">>, #{ qos => 2 }),
-    true = emqx:subscribed(<<"topic">>, <<"clientId">>),
+    true = emqx:subscribed(<<"clientId">>, <<"topic">>),
     Topics = emqx:topics(),
     lists:foreach(fun(Topic) ->
                       ?assert(lists:member(Topic, Topics))
                   end, Topics),
-    ok = emqx:unsubscribe(<<"topic">>, <<"clientId">>),
+    ok = emqx:unsubscribe(<<"topic">>),
-    ok = emqx:unsubscribe(<<"topic/1">>, <<"clientId">>),
+    ok = emqx:unsubscribe(<<"topic/1">>),
-    ok = emqx:unsubscribe(<<"topic/2">>, <<"clientId">>).
+    ok = emqx:unsubscribe(<<"topic/2">>).
 
 publish(_) ->
     Msg = emqx_message:make(ct, <<"test/pubsub">>, <<"hello">>),
@@ -85,18 +84,25 @@ dispatch_with_no_sub(_) ->
 pubsub(_) ->
     true = emqx:is_running(node()),
     Self = self(),
-    Subscriber = {Self, <<"clientId">>},
+    Subscriber = <<"clientId">>,
-    ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 1 }),
+    ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 1 }),
-    #{qos := 1} = ets:lookup_element(emqx_suboption, {<<"a/b/c">>, Subscriber}, 2),
+    #{qos := 1} = ets:lookup_element(emqx_suboption, {Self, <<"a/b/c">>}, 2),
-    #{qos := 1} = emqx:get_subopts(<<"a/b/c">>, Subscriber),
+    #{qos := 1} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>),
-    true = emqx:set_subopts(<<"a/b/c">>, Subscriber, #{qos => 0}),
+    true = emqx_broker:set_subopts(<<"a/b/c">>, #{qos => 0}),
-    #{qos := 0} = emqx:get_subopts(<<"a/b/c">>, Subscriber),
+    #{qos := 0} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>),
-    ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 2 }),
+    ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 2 }),
     %% ct:log("Emq Sub: ~p.~n", [ets:lookup(emqx_suboption, {<<"a/b/c">>, Subscriber})]),
     timer:sleep(10),
-    [{Self, <<"clientId">>}] = emqx_broker:subscribers(<<"a/b/c">>),
+    [Self] = emqx_broker:subscribers(<<"a/b/c">>),
     emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)),
-    ?assert(receive {dispatch, <<"a/b/c">>, _ } -> true; P -> ct:log("Receive Message: ~p~n",[P]) after 2 -> false end),
+    ?assert(
+        receive {dispatch, <<"a/b/c">>, _ } ->
+                true;
+            P ->
+                ct:log("Receive Message: ~p~n",[P])
+        after 2 ->
+            false
+        end),
     spawn(fun() ->
             emqx:subscribe(<<"a/b/c">>),
             emqx:subscribe(<<"c/d/e">>),
@@ -106,38 +112,15 @@ pubsub(_) ->
     timer:sleep(20),
     emqx:unsubscribe(<<"a/b/c">>).
 
-t_local_subscribe(_) ->
-    ok = emqx:subscribe(<<"$local/topic0">>),
-    ok = emqx:subscribe(<<"$local/topic1">>, <<"clientId">>),
-    ok = emqx:subscribe(<<"$local/topic2">>, <<"clientId">>, #{ qos => 2 }),
-    timer:sleep(10),
-    ?assertEqual([{self(), undefined}], emqx:subscribers("$local/topic0")),
-    ?assertEqual([{self(), <<"clientId">>}], emqx:subscribers("$local/topic1")),
-    ?assertEqual([{<<"$local/topic1">>, #{ qos => 0 }},
-                  {<<"$local/topic2">>, #{ qos => 2 }}],
-                 emqx:subscriptions({self(), <<"clientId">>})),
-    ?assertEqual(ok, emqx:unsubscribe("$local/topic0")),
-    ?assertEqual(ok, emqx:unsubscribe("$local/topic0")),
-    ?assertEqual(ok, emqx:unsubscribe("$local/topic1", <<"clientId">>)),
-    ?assertEqual(ok, emqx:unsubscribe("$local/topic2", <<"clientId">>)),
-    ?assertEqual([], emqx:subscribers("topic1")),
-    ?assertEqual([], emqx:subscriptions({self(), <<"clientId">>})).
-
 t_shared_subscribe(_) ->
-    emqx:subscribe("$local/$share/group1/topic1"),
     emqx:subscribe("$share/group2/topic2"),
     emqx:subscribe("$queue/topic3"),
     timer:sleep(10),
-    ct:log("share subscriptions: ~p~n", [emqx:subscriptions({self(), undefined})]),
+    ct:log("share subscriptions: ~p~n", [emqx:subscriptions(self())]),
-    ?assertEqual([{self(), undefined}], emqx:subscribers(<<"$local/$share/group1/topic1">>)),
+    ?assertEqual(2, length(emqx:subscriptions(self()))),
-    ?assertEqual([{<<"$local/$share/group1/topic1">>, #{qos => 0}},
-                  {<<"$queue/topic3">>, #{qos => 0}},
-                  {<<"$share/group2/topic2">>, #{qos => 0}}],
-                 lists:sort(emqx:subscriptions({self(), undefined}))),
-    emqx:unsubscribe("$local/$share/group1/topic1"),
     emqx:unsubscribe("$share/group2/topic2"),
     emqx:unsubscribe("$queue/topic3"),
-    ?assertEqual([], lists:sort(emqx:subscriptions(self()))).
+    ?assertEqual(0, length(emqx:subscriptions(self()))).
 
 'pubsub#'(_) ->
     emqx:subscribe(<<"a/#">>),

test/emqx_client_SUITE.erl

@@ -33,8 +33,7 @@
 
 all() ->
     [{group, mqttv4},
-      {group, mqttv5}
+     {group, mqttv5}].
-    ].
 
 groups() ->
     [{mqttv4, [non_parallel_tests],
@@ -48,8 +47,7 @@ groups() ->
        dollar_topics_test]},
      {mqttv5, [non_parallel_tests],
       [request_response,
-       share_sub_request_topic]}
+       share_sub_request_topic]}].
-].
 
 init_per_suite(Config) ->
     emqx_ct_broker_helpers:run_setup_steps(),

test/emqx_cm_SUITE.erl

@@ -1,3 +1,4 @@
+
 %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,21 +18,53 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
+-include("emqx.hrl").
 -include("emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
 
-all() -> [t_register_unregister_connection].
+all() -> [{group, cm}].
 
-t_register_unregister_connection(_) ->
+groups() ->
-    {ok, _} = emqx_cm_sup:start_link(),
+    [{cm, [non_parallel_tests],
-    Pid = self(),
+      [t_get_set_conn_attrs,
-    emqx_cm:register_connection(<<"conn1">>),
+       t_get_set_conn_stats,
-    emqx_cm:register_connection({<<"conn2">>, Pid}, [{port, 8080}, {ip, "192.168.0.1"}]),
+       t_lookup_conn_pid]}].
-    timer:sleep(2000),
+
-    [{<<"conn1">>, Pid}] = emqx_cm:lookup_connection(<<"conn1">>),
+init_per_suite(Config) ->
-    [{<<"conn2">>, Pid}] = emqx_cm:lookup_connection(<<"conn2">>),
+    emqx_ct_broker_helpers:run_setup_steps(),
-    Pid = emqx_cm:lookup_conn_pid(<<"conn1">>),
+    Config.
-    emqx_cm:unregister_connection(<<"conn1">>),
+
-    [] = emqx_cm:lookup_connection(<<"conn1">>),
+end_per_suite(_Config) ->
-    [{port, 8080}, {ip, "192.168.0.1"}] = emqx_cm:get_conn_attrs({<<"conn2">>, Pid}),
+    emqx_ct_broker_helpers:run_teardown_steps().
-    emqx_cm:set_conn_stats(<<"conn2">>, [[{count, 1}, {max, 2}]]),
+
-    [[{count, 1}, {max, 2}]] = emqx_cm:get_conn_stats({<<"conn2">>, Pid}).
+init_per_testcase(_TestCase, Config) ->
+    register_connection(),
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    unregister_connection(),
+    ok.
+
+t_get_set_conn_attrs(_) ->
+    ?assert(emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}])),
+    ?assert(emqx_cm:set_conn_attrs(<<"conn2">>, self(), [{port, 8080}, {ip, "192.168.0.2"}])),
+    ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs(<<"conn1">>)),
+    ?assertEqual([{port, 8080}, {ip, "192.168.0.2"}], emqx_cm:get_conn_attrs(<<"conn2">>, self())).
+
+t_get_set_conn_stats(_) ->
+    ?assert(emqx_cm:set_conn_stats(<<"conn1">>, [{count, 1}, {max, 2}])),
+    ?assert(emqx_cm:set_conn_stats(<<"conn2">>, self(), [{count, 1}, {max, 2}])),
+    ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn1">>)),
+    ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn2">>, self())).
+
+t_lookup_conn_pid(_) ->
+    ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>, self())),
+    ?assertEqual(self(), emqx_cm:lookup_conn_pid(<<"conn1">>)).
+
+register_connection() ->
+    ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>)),
+    ?assertEqual(ok, emqx_cm:register_connection(<<"conn2">>, self())).
+
+unregister_connection() ->
+    ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn1">>)),
+    ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn2">>, self())).

test/emqx_gc_SUITE.erl

@@ -0,0 +1,57 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_gc_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() ->
+    [t_init, t_run, t_info, t_reset].
+
+t_init(_) ->
+    ?assertEqual(undefined, emqx_gc:init(false)),
+    GC1 = emqx_gc:init(#{count => 10, bytes => 0}),
+    ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC1)),
+    GC2 = emqx_gc:init(#{count => 0, bytes => 10}),
+    ?assertEqual(#{oct => {10, 10}}, emqx_gc:info(GC2)),
+    GC3 = emqx_gc:init(#{count => 10, bytes => 10}),
+    ?assertEqual(#{cnt => {10, 10}, oct => {10, 10}}, emqx_gc:info(GC3)).
+
+t_run(_) ->
+    GC = emqx_gc:init(#{count => 10, bytes => 10}),
+    ?assertEqual({true, GC}, emqx_gc:run(1, 1000, GC)),
+    ?assertEqual({true, GC}, emqx_gc:run(1000, 1, GC)),
+    {false, GC1} = emqx_gc:run(1, 1, GC),
+    ?assertEqual(#{cnt => {10, 9}, oct => {10, 9}}, emqx_gc:info(GC1)),
+    {false, GC2} = emqx_gc:run(2, 2, GC1),
+    ?assertEqual(#{cnt => {10, 7}, oct => {10, 7}}, emqx_gc:info(GC2)),
+    {false, GC3} = emqx_gc:run(3, 3, GC2),
+    ?assertEqual(#{cnt => {10, 4}, oct => {10, 4}}, emqx_gc:info(GC3)),
+    ?assertEqual({true, GC}, emqx_gc:run(4, 4, GC3)).
+
+t_info(_) ->
+    ?assertEqual(undefined, emqx_gc:info(undefined)),
+    GC = emqx_gc:init(#{count => 10, bytes => 0}),
+    ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC)).
+
+t_reset(_) ->
+    ?assertEqual(undefined, emqx_gc:reset(undefined)),
+    GC = emqx_gc:init(#{count => 10, bytes => 10}),
+    {false, GC1} = emqx_gc:run(5, 5, GC),
+    ?assertEqual(#{cnt => {10, 5}, oct => {10, 5}}, emqx_gc:info(GC1)),
+    ?assertEqual(GC, emqx_gc:reset(GC1)).
+

test/emqx_gc_tests.erl

@@ -1,53 +0,0 @@
-%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-
--module(emqx_gc_tests).
-
--include_lib("eunit/include/eunit.hrl").
-
-trigger_by_cnt_test() ->
-    Args = #{count => 2, bytes => 0},
-    ok = emqx_gc:init(Args),
-    ok = emqx_gc:inc(1, 1000),
-    St1 = inspect(),
-    ?assertMatch({_, Remain} when Remain > 0, maps:get(cnt, St1)),
-    ok = emqx_gc:inc(2, 2),
-    St2 = inspect(),
-    ok = emqx_gc:inc(0, 2000),
-    St3 = inspect(),
-    ?assertEqual(St2, St3),
-    ?assertMatch({N, N}, maps:get(cnt, St2)),
-    ?assertNot(maps:is_key(oct, St2)),
-    ok.
-
-trigger_by_oct_test() ->
-    Args = #{count => 2, bytes => 2},
-    ok = emqx_gc:init(Args),
-    ok = emqx_gc:inc(1, 1),
-    St1 = inspect(),
-    ?assertMatch({_, Remain} when Remain > 0, maps:get(oct, St1)),
-    ok = emqx_gc:inc(2, 2),
-    St2 = inspect(),
-    ?assertMatch({N, N}, maps:get(oct, St2)),
-    ?assertMatch({M, M}, maps:get(cnt, St2)),
-    ok.
-
-disabled_test() ->
-    Args = #{count => -1, bytes => false},
-    ok = emqx_gc:init(Args),
-    ok = emqx_gc:inc(1, 1),
-    ?assertEqual(#{}, inspect()),
-    ok.
-
-inspect() -> erlang:get(emqx_gc).

test/emqx_pd_SUITE.erl

@@ -0,0 +1,31 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_pd_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() -> [update_counter].
+
+update_counter(_) ->
+    ?assertEqual(undefined, emqx_pd:update_counter(bytes, 1)),
+    ?assertEqual(1, emqx_pd:update_counter(bytes, 1)),
+    ?assertEqual(2, emqx_pd:update_counter(bytes, 1)),
+    ?assertEqual(3, emqx_pd:get_counter(bytes)),
+    ?assertEqual(3, emqx_pd:reset_counter(bytes)),
+    ?assertEqual(0, emqx_pd:get_counter(bytes)).
+

test/emqx_pmon_SUITE.erl

@@ -0,0 +1,48 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_pmon_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() ->
+    [t_monitor, t_find, t_erase].
+
+t_monitor(_) ->
+    PMon = emqx_pmon:new(),
+    PMon1 = emqx_pmon:monitor(self(), PMon),
+    ?assertEqual(1, emqx_pmon:count(PMon1)),
+    PMon2 = emqx_pmon:demonitor(self(), PMon1),
+    ?assertEqual(0, emqx_pmon:count(PMon2)).
+
+t_find(_) ->
+    PMon = emqx_pmon:new(),
+    PMon1 = emqx_pmon:monitor(self(), val, PMon),
+    ?assertEqual(1, emqx_pmon:count(PMon1)),
+    ?assertEqual({ok, val}, emqx_pmon:find(self(), PMon1)),
+    PMon2 = emqx_pmon:erase(self(), PMon1),
+    ?assertEqual(error, emqx_pmon:find(self(), PMon2)).
+
+t_erase(_) ->
+    PMon = emqx_pmon:new(),
+    PMon1 = emqx_pmon:monitor(self(), val, PMon),
+    PMon2 = emqx_pmon:erase(self(), PMon1),
+    ?assertEqual(0, emqx_pmon:count(PMon2)),
+    {Items, PMon3} = emqx_pmon:erase_all([self()], PMon1),
+    ?assertEqual([{self(), val}], Items),
+    ?assertEqual(0, emqx_pmon:count(PMon3)).
+

test/emqx_protocol_SUITE.erl

@@ -62,6 +62,7 @@ init_per_suite(Config) ->
         {App, SchemaFile, ConfigFile}
             <- [{emqx, deps_path(emqx, "priv/emqx.schema"),
                        deps_path(emqx, "etc/emqx.conf")}]],
+    emqx_zone:set_env(external, max_topic_alias, 20),
     Config.
 
 end_per_suite(_Config) ->
@@ -162,6 +163,82 @@ connect_v5(_) ->
                                 raw_recv_parse(Data, ?MQTT_PROTO_V5)
                     end),
 
+    % topic alias = 0
+    with_connection(fun([Sock]) ->
+                            emqx_client_sock:send(Sock,
+                                                  raw_send_serialize(
+                                                      ?CONNECT_PACKET(
+                                                          #mqtt_packet_connect{
+                                                              proto_ver  = ?MQTT_PROTO_V5,
+                                                              properties =
+                                                                  #{'Topic-Alias-Maximum' => 10}}),
+                                                      #{version => ?MQTT_PROTO_V5}
+                                                  )),
+                            {ok, Data} = gen_tcp:recv(Sock, 0),
+                            {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0,
+                                                 #{'Topic-Alias-Maximum' := 20}), _} =
+                                raw_recv_parse(Data, ?MQTT_PROTO_V5),
+
+                            emqx_client_sock:send(Sock,
+                                                  raw_send_serialize(
+                                                      ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 0}, <<"hello">>),
+                                                      #{version => ?MQTT_PROTO_V5}
+                                                  )),
+
+                            {ok, Data2} = gen_tcp:recv(Sock, 0),
+                            {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5)
+                    end),
+
+    % topic alias maximum
+    with_connection(fun([Sock]) ->
+                            emqx_client_sock:send(Sock,
+                                                  raw_send_serialize(
+                                                      ?CONNECT_PACKET(
+                                                          #mqtt_packet_connect{
+                                                              proto_ver  = ?MQTT_PROTO_V5,
+                                                              properties =
+                                                                  #{'Topic-Alias-Maximum' => 10}}),
+                                                      #{version => ?MQTT_PROTO_V5}
+                                                  )),
+                            {ok, Data} = gen_tcp:recv(Sock, 0),
+                            {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0,
+                                                 #{'Topic-Alias-Maximum' := 20}), _} =
+                                raw_recv_parse(Data, ?MQTT_PROTO_V5),
+
+                            emqx_client_sock:send(Sock, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh  => 1,
+                                                                                                                  qos => ?QOS_2,
+                                                                                                                  rap => 0,
+                                                                                                                  nl  => 0,
+                                                                                                                  rc  => 0}}]),
+                                                                            #{version => ?MQTT_PROTO_V5})),
+
+                            {ok, Data2} = gen_tcp:recv(Sock, 0),
+                            {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5),
+
+                            emqx_client_sock:send(Sock,
+                                                  raw_send_serialize(
+                                                      ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 15}, <<"hello">>),
+                                                      #{version => ?MQTT_PROTO_V5}
+                                                  )),
+
+                            {ok, Data3} = gen_tcp:recv(Sock, 0),
+
+                            {ok, ?PUBACK_PACKET(1, 0), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5),
+
+                            {ok, Data4} = gen_tcp:recv(Sock, 0),
+
+                            {ok, ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, _, <<"hello">>), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5),
+
+                            emqx_client_sock:send(Sock,
+                                                  raw_send_serialize(
+                                                      ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 2, #{'Topic-Alias' => 21}, <<"hello">>),
+                                                      #{version => ?MQTT_PROTO_V5}
+                                                  )),
+
+                            {ok, Data5} = gen_tcp:recv(Sock, 0),
+                            {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data5, ?MQTT_PROTO_V5)
+                    end),
+
     % test clean start
     with_connection(fun([Sock]) ->
                             emqx_client_sock:send(Sock,

test/emqx_router_SUITE.erl

@@ -21,17 +21,16 @@
 -compile(nowarn_export_all).
 
 -define(R, emqx_router).
--define(TABS, [emqx_route, emqx_trie, emqx_trie_node]).
 
 all() ->
     [{group, route}].
 
 groups() ->
     [{route, [sequence],
-      [add_del_route,
+      [t_add_delete,
-       match_routes,
+       t_do_add_delete,
-       has_routes,
+       t_match_routes,
-       router_add_del]}].
+       t_has_routes]}].
 
 init_per_suite(Config) ->
     emqx_ct_broker_helpers:run_setup_steps(),
@@ -47,77 +46,47 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     clear_tables().
 
-add_del_route(_) ->
+t_add_delete(_) ->
-    From = {self(), make_ref()},
+    ?R:add_route(<<"a/b/c">>, node()),
-    ?R:add_route(From, <<"a/b/c">>, node()),
+    ?R:add_route(<<"a/b/c">>, node()),
-    timer:sleep(1),
+    ?R:add_route(<<"a/+/b">>, node()),
-
-    ?R:add_route(From, <<"a/b/c">>, node()),
-    timer:sleep(1),
-
-    ?R:add_route(From, <<"a/+/b">>, node()),
-    ct:log("Topics: ~p ~n", [emqx_topic:wildcard(<<"a/+/b">>)]),
-    timer:sleep(1),
-
     ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())),
 
-    ?R:del_route(From, <<"a/b/c">>, node()),
+    ?R:delete_route(<<"a/b/c">>),
+    ?R:delete_route(<<"a/+/b">>, node()),
+    ?assertEqual([], ?R:topics()).
 
-    ?R:del_route(From, <<"a/+/b">>, node()),
+t_do_add_delete(_) ->
-    timer:sleep(120),
+    ?R:do_add_route(<<"a/b/c">>, node()),
-    ?assertEqual([], lists:sort(?R:topics())).
+    ?R:do_add_route(<<"a/b/c">>, node()),
+    ?R:do_add_route(<<"a/+/b">>, node()),
+    ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())),
 
-match_routes(_) ->
+    ?R:do_delete_route(<<"a/b/c">>, node()),
-    From = {self(), make_ref()},
+    ?R:do_delete_route(<<"a/+/b">>),
-    ?R:add_route(From, <<"a/b/c">>, node()),
+    ?assertEqual([], ?R:topics()).
-    ?R:add_route(From, <<"a/+/c">>, node()),
+
-    ?R:add_route(From, <<"a/b/#">>, node()),
+t_match_routes(_) ->
-    ?R:add_route(From, <<"#">>, node()),
+    ?R:add_route(<<"a/b/c">>, node()),
-    timer:sleep(1000),
+    ?R:add_route(<<"a/+/c">>, node()),
+    ?R:add_route(<<"a/b/#">>, node()),
+    ?R:add_route(<<"#">>, node()),
     ?assertEqual([#route{topic = <<"#">>, dest = node()},
                   #route{topic = <<"a/+/c">>, dest = node()},
                   #route{topic = <<"a/b/#">>, dest = node()},
                   #route{topic = <<"a/b/c">>, dest = node()}],
-                 lists:sort(?R:match_routes(<<"a/b/c">>))).
+                 lists:sort(?R:match_routes(<<"a/b/c">>))),
+    ?R:delete_route(<<"a/b/c">>, node()),
+    ?R:delete_route(<<"a/+/c">>, node()),
+    ?R:delete_route(<<"a/b/#">>, node()),
+    ?R:delete_route(<<"#">>, node()),
+    ?assertEqual([], lists:sort(?R:match_routes(<<"a/b/c">>))).
 
-has_routes(_) ->
+t_has_routes(_) ->
-    From = {self(), make_ref()},
+    ?R:add_route(<<"devices/+/messages">>, node()),
-    ?R:add_route(From, <<"devices/+/messages">>, node()),
+    ?assert(?R:has_routes(<<"devices/+/messages">>)),
-    timer:sleep(200),
+    ?R:delete_route(<<"devices/+/messages">>).
-    ?assert(?R:has_routes(<<"devices/+/messages">>)).
 
 clear_tables() ->
-    lists:foreach(fun mnesia:clear_table/1, ?TABS).
+    lists:foreach(fun mnesia:clear_table/1, [emqx_route, emqx_trie, emqx_trie_node]).
-
-router_add_del(_) ->
-    ?R:add_route(<<"#">>),
-    ?R:add_route(<<"a/b/c">>, node()),
-    ?R:add_route(<<"+/#">>),
-    Routes = [R1, R2 | _] = [
-            #route{topic = <<"#">>,     dest = node()},
-            #route{topic = <<"+/#">>,   dest = node()},
-            #route{topic = <<"a/b/c">>, dest = node()}],
-    timer:sleep(500),
-    ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))),
-
-    ?R:print_routes(<<"a/b/c">>),
-
-    %% Batch Add
-    lists:foreach(fun(R) -> ?R:add_route(R) end, Routes),
-    ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))),
-
-    %% Del
-    ?R:del_route(<<"a/b/c">>, node()),
-    timer:sleep(500),
-    [R1, R2] = lists:sort(?R:match_routes(<<"a/b/c">>)),
-    {atomic, []} = mnesia:transaction(fun emqx_trie:lookup/1, [<<"a/b/c">>]),
-
-    %% Batch Del
-    R3 = #route{topic = <<"#">>, dest = 'a@127.0.0.1'},
-    ?R:add_route(R3),
-    ?R:del_route(<<"#">>),
-    ?R:del_route(R2),
-    ?R:del_route(R3),
-    timer:sleep(500),
-    [] = lists:sort(?R:match_routes(<<"a/b/c">>)).
 

test/emqx_sequence_SUITE.erl

@@ -0,0 +1,38 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_sequence_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-import(emqx_sequence, [nextval/2, reclaim/2]).
+
+all() ->
+    [sequence_generate].
+
+sequence_generate(_) ->
+    ok = emqx_sequence:create(seqtab),
+    ?assertEqual(1, nextval(seqtab, key)),
+    ?assertEqual(2, nextval(seqtab, key)),
+    ?assertEqual(3, nextval(seqtab, key)),
+    ?assertEqual(2, reclaim(seqtab, key)),
+    ?assertEqual(1, reclaim(seqtab, key)),
+    ?assertEqual(0, reclaim(seqtab, key)),
+    ?assertEqual(false, ets:member(seqtab, key)),
+    ?assertEqual(1, nextval(seqtab, key)),
+    ?assert(emqx_sequence:delete(seqtab)).
+

test/emqx_session_SUITE.erl

@@ -53,21 +53,10 @@ t_session_all(_) ->
     emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 2}}]),
     emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 1}}]),
     timer:sleep(200),
-    [{<<"topic">>, _}] = emqx:subscriptions({SPid, <<"ClientId">>}),
+    [{<<"topic">>, _}] = emqx:subscriptions(SPid),
     emqx_session:publish(SPid, 1, Message1),
     timer:sleep(200),
     {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid),
-    emqx_session:puback(SPid, 2),
-    emqx_session:puback(SPid, 3, reasoncode),
-    emqx_session:pubrec(SPid, 4),
-    emqx_session:pubrec(SPid, 5, reasoncode),
-    emqx_session:pubrel(SPid, 6, reasoncode),
-    emqx_session:pubcomp(SPid, 7, reasoncode),
-    timer:sleep(200),
-    2 = emqx_metrics:val('packets/puback/missed'),
-    2 = emqx_metrics:val('packets/pubrec/missed'),
-    1 = emqx_metrics:val('packets/pubrel/missed'),
-    1 = emqx_metrics:val('packets/pubcomp/missed'),
     Attrs = emqx_session:attrs(SPid),
     Info = emqx_session:info(SPid),
     Stats = emqx_session:stats(SPid),
@@ -76,5 +65,5 @@ t_session_all(_) ->
     1 = proplists:get_value(subscriptions_count, Stats),
     emqx_session:unsubscribe(SPid, [<<"topic">>]),
     timer:sleep(200),
-    [] = emqx:subscriptions({SPid, <<"clientId">>}),
+    [] = emqx:subscriptions(SPid),
     emqx_mock_client:close_session(ConnPid).

test/emqx_sm_SUITE.erl

@@ -15,35 +15,78 @@
 -module(emqx_sm_SUITE).
 
 -include("emqx.hrl").
+-include_lib("eunit/include/eunit.hrl").
 
 -compile(export_all).
 -compile(nowarn_export_all).
 
-all() -> [t_open_close_session].
+-define(ATTRS, #{clean_start         => true,
-
-t_open_close_session(_) ->
-    emqx_ct_broker_helpers:run_setup_steps(),
-    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
-    Attrs = #{clean_start         => true,
                  client_id           => <<"client">>,
-              conn_pid            => ClientPid,
                  zone                => internal,
                  username            => <<"emqx">>,
                  expiry_interval     => 0,
                  max_inflight        => 0,
                  topic_alias_maximum => 0,
-              will_msg            => undefined},
+                 will_msg            => undefined}).
-    {ok, SPid} = emqx_sm:open_session(Attrs),
+
-    [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>),
+all() -> [{group, sm}].
-    SPid = emqx_sm:lookup_session_pid(<<"client">>),
+
-    {ok, NewConnPid} = emqx_mock_client:start_link(<<"client">>),
+groups() ->
-    {ok, SPid, true} = emqx_sm:open_session(Attrs#{clean_start => false, conn_pid => NewConnPid}),
+    [{sm, [non_parallel_tests],
-    [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>),
+      [t_open_close_session,
-    SAttrs = emqx_sm:get_session_attrs({<<"client">>, SPid}),
+       t_resume_session,
-    <<"client">> = proplists:get_value(client_id, SAttrs),
+       t_discard_session,
-    Session = {<<"client">>, SPid},
+       t_register_unregister_session,
-    emqx_sm:set_session_stats(Session, {open, true}),
+       t_get_set_session_attrs,
-    {open, true} = emqx_sm:get_session_stats(Session),
+       t_get_set_session_stats,
-    ok = emqx_sm:close_session(SPid),
+       t_lookup_session_pids]}].
-    [] = emqx_sm:lookup_session(<<"client">>),
+
+init_per_suite(Config) ->
+    emqx_ct_broker_helpers:run_setup_steps(),
+    Config.
+
+end_per_suite(_Config) ->
     emqx_ct_broker_helpers:run_teardown_steps().
+
+t_open_close_session(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual(ok, emqx_sm:close_session(SPid)).
+
+t_resume_session(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual({ok, SPid}, emqx_sm:resume_session(<<"client">>, ?ATTRS#{conn_pid => ClientPid})).
+
+t_discard_session(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client1">>),
+    {ok, _SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual(ok, emqx_sm:discard_session(<<"client1">>)).
+
+t_register_unregister_session(_) ->
+    Pid = self(),
+    {ok, _ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    ?assertEqual(ok, emqx_sm:register_session(<<"client">>)),
+    ?assertEqual(ok, emqx_sm:register_session(<<"client">>, Pid)),
+    ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>)),
+    ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>), Pid).
+
+t_get_set_session_attrs(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, [?ATTRS#{conn_pid => ClientPid}])),
+    ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, SPid, [?ATTRS#{conn_pid => ClientPid}])),
+    [SAttr] = emqx_sm:get_session_attrs(<<"client">>, SPid),
+    ?assertEqual(<<"client">>, maps:get(client_id, SAttr)).
+
+t_get_set_session_stats(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, [{inflight, 10}])),
+    ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}])),
+    ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)).
+
+t_lookup_session_pids(_) ->
+    {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>),
+    {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}),
+    ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)).

test/emqx_stats_tests.erl

@@ -75,10 +75,10 @@ helper_test_() ->
                         with_proc(fun() -> TestF(CbModule, CbFun) end, TickMs)
                 end
         end,
-    [{"emqx_broker_helper", MkTestFun(emqx_broker_helper, stats_fun)},
+    [{"emqx_broker", MkTestFun(emqx_broker, stats_fun)},
      {"emqx_sm", MkTestFun(emqx_sm, stats_fun)},
      {"emqx_router_helper", MkTestFun(emqx_router_helper, stats_fun)},
-     {"emqx_cm", MkTestFun(emqx_cm, update_conn_stats)}
+     {"emqx_cm", MkTestFun(emqx_cm, stats_fun)}
     ].
 
 with_proc(F) ->

test/emqx_tables_SUITE.erl

@@ -20,7 +20,7 @@
 all() -> [t_new].
 
 t_new(_) ->
-    TId = emqx_tables:new(test_table, [{read_concurrency, true}]),
+    ok = emqx_tables:new(test_table, [{read_concurrency, true}]),
-    ets:insert(TId, {loss, 100}),
+    ets:insert(test_table, {key, 100}),
-    TId = emqx_tables:new(test_table, [{read_concurrency, true}]),
+    ok = emqx_tables:new(test_table, [{read_concurrency, true}]),
-    100 = ets:lookup_element(TId, loss, 2).
+    100 = ets:lookup_element(test_table, key, 2).

test/emqx_trie_SUITE.erl

@@ -24,7 +24,7 @@
 -define(TRIE_TABS, [emqx_trie, emqx_trie_node]).
 
 all() ->
-    [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3].
+    [t_mnesia, t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3].
 
 init_per_suite(Config) ->
     application:load(emqx),
@@ -42,41 +42,44 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     clear_tables().
 
+t_mnesia(_) ->
+    ok = ?TRIE:mnesia(copy).
+
 t_insert(_) ->
     TN = #trie_node{node_id = <<"sensor">>,
                     edge_count = 3,
                     topic = <<"sensor">>,
                     flags = undefined},
-    {atomic, [TN]} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"sensor/1/metric/2">>),
               ?TRIE:insert(<<"sensor/+/#">>),
               ?TRIE:insert(<<"sensor/#">>),
               ?TRIE:insert(<<"sensor">>),
               ?TRIE:insert(<<"sensor">>),
               ?TRIE:lookup(<<"sensor">>)
-            end).
+          end,
+    ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)).
 
 t_match(_) ->
     Machted = [<<"sensor/+/#">>, <<"sensor/#">>],
-    {atomic, Machted} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"sensor/1/metric/2">>),
               ?TRIE:insert(<<"sensor/+/#">>),
               ?TRIE:insert(<<"sensor/#">>),
               ?TRIE:match(<<"sensor/1">>)
-            end).
+            end,
+    ?assertEqual({atomic, Machted}, mnesia:transaction(Fun)).
 
 t_match2(_) ->
     Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []},
-    {atomic, Matched} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"#">>),
               ?TRIE:insert(<<"+/#">>),
               ?TRIE:insert(<<"+/+/#">>),
               {?TRIE:match(<<"a/b/c">>),
                ?TRIE:match(<<"$SYS/broker/zenmq">>)}
-            end).
+          end,
+    ?assertEqual({atomic, Matched}, mnesia:transaction(Fun)).
 
 t_match3(_) ->
     Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>],
@@ -91,8 +94,7 @@ t_delete(_) ->
                     edge_count = 2,
                     topic = undefined,
                     flags = undefined},
-    {atomic, [TN]} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"sensor/1/#">>),
               ?TRIE:insert(<<"sensor/1/metric/2">>),
               ?TRIE:insert(<<"sensor/1/metric/3">>),
@@ -100,24 +102,23 @@ t_delete(_) ->
               ?TRIE:delete(<<"sensor/1/metric">>),
               ?TRIE:delete(<<"sensor/1/metric">>),
               ?TRIE:lookup(<<"sensor/1">>)
-            end).
+          end,
+    ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)).
 
 t_delete2(_) ->
-    {atomic, {[], []}} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"sensor">>),
               ?TRIE:insert(<<"sensor/1/metric/2">>),
               ?TRIE:insert(<<"sensor/1/metric/3">>),
               ?TRIE:delete(<<"sensor">>),
               ?TRIE:delete(<<"sensor/1/metric/2">>),
               ?TRIE:delete(<<"sensor/1/metric/3">>),
-                {?TRIE:lookup(<<"sensor">>),
+              {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)}
-                 ?TRIE:lookup(<<"sensor/1">>)}
+          end,
-            end).
+    ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)).
 
 t_delete3(_) ->
-    {atomic, {[], []}} = mnesia:transaction(
+    Fun = fun() ->
-            fun() ->
               ?TRIE:insert(<<"sensor/+">>),
               ?TRIE:insert(<<"sensor/+/metric/2">>),
               ?TRIE:insert(<<"sensor/+/metric/3">>),
@@ -127,7 +128,8 @@ t_delete3(_) ->
               ?TRIE:delete(<<"sensor/+">>),
               ?TRIE:delete(<<"sensor/+/unknown">>),
               {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)}
-            end).
+          end,
+    ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)).
 
 clear_tables() ->
     lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS).