Merge remote-tracking branch 'origin/develop'

.gitignore

@@ -38,3 +38,4 @@ cuttlefish
 rebar.lock
 xrefr
 erlang.mk
+*.coverdata

Makefile

@@ -20,14 +20,14 @@ ERLC_OPTS += +debug_info -DAPPLICATION=emqx
 BUILD_DEPS = cuttlefish
 dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1
 
-#TEST_DEPS = emqx_ct_helplers
+TEST_DEPS = meck
-#dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers
+dep_meck = hex-emqx 0.8.13
 
 TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx
 
 EUNIT_OPTS = verbose
 
-# CT_SUITES = emqx_frame
+# CT_SUITES = emqx_bridge
 ## emqx_trie emqx_router emqx_frame emqx_mqtt_compat
 
 CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
@@ -37,7 +37,8 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
 			emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \
 			emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \
 			emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \
-			emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message
+			emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message emqx_os_mon \
+      emqx_vm_mon emqx_alarm_handler
 
 CT_NODE_NAME = emqxct@127.0.0.1
 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME)
@@ -96,17 +97,24 @@ rebar-deps:
 	@rebar3 get-deps
 
 rebar-eunit: $(CUTTLEFISH_SCRIPT)
-	@rebar3 eunit
+	@rebar3 eunit -v
 
 rebar-compile:
 	@rebar3 compile
 
-rebar-ct: app.config
+rebar-ct-setup: app.config
 	@rebar3 as test compile
 	@ln -s -f '../../../../etc' _build/test/lib/emqx/
 	@ln -s -f '../../../../data' _build/test/lib/emqx/
+
+rebar-ct: rebar-ct-setup
 	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',')
 
+## Run one single CT with rebar3
+## e.g. make ct-one-suite suite=emqx_bridge
+ct-one-suite: rebar-ct-setup
+	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE
+
 rebar-clean:
 	@rebar3 clean
 

etc/emqx.conf

@@ -1596,28 +1596,6 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ##--------------------------------------------------------------------
 ## Bridges to aws
 ##--------------------------------------------------------------------
-## Start type of the bridge.
-##
-## Value: enum
-## manual
-## auto
-## bridge.aws.start_type = manual
-
-## Bridge reconnect time.
-##
-## Value: Duration
-## Default: 30 seconds
-## bridge.aws.reconnect_interval = 30s
-
-## Retry interval for bridge QoS1 message delivering.
-##
-## Value: Duration
-## bridge.aws.retry_interval = 20s
-
-## Inflight size.
-##
-## Value: Integer
-## bridge.aws.max_inflight = 32
 
 ## Bridge address: node name for local bridge, host:port for remote.
 ##
@@ -1662,66 +1640,12 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Value: String
 ## bridge.aws.mountpoint = bridge/aws/${node}/
 
-## Ping interval of a down bridge.
-##
-## Value: Duration
-## Default: 10 seconds
-## bridge.aws.keepalive = 60s
-
 ## Forward message topics
 ##
 ## Value: String
 ## Example: topic1/#,topic2/#
 ## bridge.aws.forwards = topic1/#,topic2/#
 
-## Subscriptions of the bridge topic.
-##
-## Value: String
-## bridge.aws.subscription.1.topic = cmd/topic1
-
-## Subscriptions of the bridge qos.
-##
-## Value: Number
-## bridge.aws.subscription.1.qos = 1
-
-## Subscriptions of the bridge topic.
-##
-## Value: String
-## bridge.aws.subscription.2.topic = cmd/topic2
-
-## Subscriptions of the bridge qos.
-##
-## Value: Number
-## bridge.aws.subscription.2.qos = 1
-
-## If enabled, queue would be written into disk more quickly.
-## However, If disabled, some message would be dropped in
-## the situation emqx crashed.
-##
-## Value: on | off
-## bridge.aws.queue.mem_cache = on
-
-## Batch size for buffer queue stored
-##
-## Value: Integer
-## default: 1000
-## bridge.aws.queue.batch_size = 1000
-
-## Base directory for replayq to store messages on disk
-## If this config entry is missing or set to undefined,
-## replayq works in a mem-only manner. If the config
-## entry was set to `bridge.aws.mqueue_type = memory`
-## this config entry would have no effect on mqueue
-##
-## Value: String
-## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/
-
-## Replayq segment size
-##
-## Value: Bytesize
-
-## bridge.aws.queue.replayq_seg_bytes = 10MB
-
 ## Bribge to remote server via SSL.
 ##
 ## Value: on | off
@@ -1747,36 +1671,89 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Value: String
 ## bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
 
+## Ping interval of a down bridge.
+##
+## Value: Duration
+## Default: 10 seconds
+## bridge.aws.keepalive = 60s
+
 ## TLS versions used by the bridge.
 ##
 ## Value: String
 ## bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1
 
-##--------------------------------------------------------------------
+## Subscriptions of the bridge topic.
-## Bridges to azure
+##
-##--------------------------------------------------------------------
+## Value: String
+## bridge.aws.subscription.1.topic = cmd/topic1
+
+## Subscriptions of the bridge qos.
+##
+## Value: Number
+## bridge.aws.subscription.1.qos = 1
+
+## Subscriptions of the bridge topic.
+##
+## Value: String
+## bridge.aws.subscription.2.topic = cmd/topic2
+
+## Subscriptions of the bridge qos.
+##
+## Value: Number
+## bridge.aws.subscription.2.qos = 1
+
 ## Start type of the bridge.
 ##
 ## Value: enum
 ## manual
 ## auto
-## bridge.azure.start_type = manual
+## bridge.aws.start_type = manual
-
-## Bridge reconnect count.
-##
-## Value: Number
-## bridge.azure.reconnect_count = 10
 
 ## Bridge reconnect time.
 ##
 ## Value: Duration
 ## Default: 30 seconds
-## bridge.azure.reconnect_time = 30s
+## bridge.aws.reconnect_interval = 30s
 
 ## Retry interval for bridge QoS1 message delivering.
 ##
 ## Value: Duration
-## bridge.azure.retry_interval = 20s
+## bridge.aws.retry_interval = 20s
+
+## Inflight size.
+##
+## Value: Integer
+## bridge.aws.max_inflight_batches = 32
+
+## Max number of messages to collect in a batch for
+## each send call towards emqx_bridge_connect
+##
+## Value: Integer
+## default: 32
+## bridge.aws.queue.batch_count_limit = 32
+
+## Max number of bytes to collect in a batch for each
+## send call towards emqx_bridge_connect
+##
+## Value: Bytesize
+## default: 1000M
+## bridge.aws.queue.batch_bytes_limit = 1000MB
+
+## Base directory for replayq to store messages on disk
+## If this config entry is missing or set to undefined,
+## replayq works in a mem-only manner.
+##
+## Value: String
+## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/
+
+## Replayq segment size
+##
+## Value: Bytesize
+## bridge.aws.queue.replayq_seg_bytes = 10MB
+
+##--------------------------------------------------------------------
+## Bridges to azure
+##--------------------------------------------------------------------
 
 ## Bridge address: node name for local bridge, host:port for remote.
 ##
@@ -1819,13 +1796,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Mountpoint of the bridge.
 ##
 ## Value: String
-## bridge.azure.mountpoint = bridge/azure/${node}/
+## bridge.azure.mountpoint = bridge/aws/${node}/
-
-## Ping interval of a down bridge.
-##
-## Value: Duration
-## Default: 10 seconds
-## bridge.azure.keepalive = 10s
 
 ## Forward message topics
 ##
@@ -1833,10 +1804,46 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Example: topic1/#,topic2/#
 ## bridge.azure.forwards = topic1/#,topic2/#
 
+## Bribge to remote server via SSL.
+##
+## Value: on | off
+## bridge.azure.ssl = off
+
+## PEM-encoded CA certificates of the bridge.
+##
+## Value: File
+## bridge.azure.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem
+
+## Client SSL Certfile of the bridge.
+##
+## Value: File
+## bridge.azure.certfile = {{ platform_etc_dir }}/certs/client-cert.pem
+
+## Client SSL Keyfile of the bridge.
+##
+## Value: File
+## bridge.azure.keyfile = {{ platform_etc_dir }}/certs/client-key.pem
+
+## SSL Ciphers used by the bridge.
+##
+## Value: String
+## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
+
+## Ping interval of a down bridge.
+##
+## Value: Duration
+## Default: 10 seconds
+## bridge.azure.keepalive = 60s
+
+## TLS versions used by the bridge.
+##
+## Value: String
+## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1
+
 ## Subscriptions of the bridge topic.
 ##
 ## Value: String
-## bridge.azure.subscription.1.topic = $share/cmd/topic1
+## bridge.azure.subscription.1.topic = cmd/topic1
 
 ## Subscriptions of the bridge qos.
 ##
@@ -1846,27 +1853,50 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Subscriptions of the bridge topic.
 ##
 ## Value: String
-## bridge.azure.subscription.2.topic = $share/cmd/topic2
+## bridge.azure.subscription.2.topic = cmd/topic2
 
 ## Subscriptions of the bridge qos.
 ##
 ## Value: Number
 ## bridge.azure.subscription.2.qos = 1
 
-## Batch size for buffer queue stored
+## Start type of the bridge.
+##
+## Value: enum
+## manual
+## auto
+## bridge.azure.start_type = manual
+
+## Bridge reconnect time.
+##
+## Value: Duration
+## Default: 30 seconds
+## bridge.azure.reconnect_interval = 30s
+
+## Retry interval for bridge QoS1 message delivering.
+##
+## Value: Duration
+## bridge.azure.retry_interval = 20s
+
+## Inflight size.
 ##
 ## Value: Integer
-## default: 1000
+## bridge.azure.max_inflight_batches = 32
-## bridge.azure.queue.batch_size = 1000
+
+## Maximum number of messages in one batch when sending to remote borkers
+## NOTE: when bridging via MQTT connection to remote broker, this config is only
+##       used for internal message passing optimization as the underlying MQTT
+##       protocol does not supports batching.
+##
+## Value: Integer
+## default: 32
+## bridge.azure.queue.batch_size = 32
 
 ## Base directory for replayq to store messages on disk
 ## If this config entry is missing or set to undefined,
-## replayq works in a mem-only manner. If the config
+## replayq works in a mem-only manner.
-## entry was set to `bridge.aws.mqueue_type = memory`
-## this config entry would have no effect on mqueue
 ##
 ## Value: String
-## Default: {{ platform_data_dir }}/emqx_aws_bridge/
 ## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/
 
 ## Replayq segment size
@@ -1874,30 +1904,6 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G
 ## Value: Bytesize
 ## bridge.azure.queue.replayq_seg_bytes = 10MB
 
-## PEM-encoded CA certificates of the bridge.
-##
-## Value: File
-## bridge.azure.cacertfile = cacert.pem
-
-## Client SSL Certfile of the bridge.
-##
-## Value: File
-## bridge.azure.certfile = cert.pem
-
-## Client SSL Keyfile of the bridge.
-##
-## Value: File
-## bridge.azure.keyfile = key.pem
-
-## SSL Ciphers used by the bridge.
-##
-## Value: String
-## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
-
-## TLS versions used by the bridge.
-##
-## Value: String
-## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1
 
 ##--------------------------------------------------------------------
 ## Modules
@@ -2049,4 +2055,61 @@ sysmon.busy_port = false
 ## Value: true | false
 sysmon.busy_dist_port = true
 
+## The time interval for the periodic cpu check
+##
+## Value: Duration
+## -h: hour, e.g. '2h' for 2 hours
+## -m: minute, e.g. '5m' for 5 minutes
+## -s: second, e.g. '30s' for 30 seconds
+##
+## Default: 60s
+os_mon.cpu_check_interval = 60s
+
+## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set.
+##
+## Default: 80%
+os_mon.cpu_high_watermark = 80%
+
+## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear.
+##
+## Default: 60%
+os_mon.cpu_low_watermark = 60%
+
+## The time interval for the periodic memory check
+##
+## Value: Duration
+## -h: hour, e.g. '2h' for 2 hours
+## -m: minute, e.g. '5m' for 5 minutes
+## -s: second, e.g. '30s' for 30 seconds
+##
+## Default: 60s
+os_mon.mem_check_interval = 60s
+
+## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set.
+##
+## Default: 70%
+os_mon.sysmem_high_watermark = 70%
+
+## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set.
+##
+## Default: 5%
+os_mon.procmem_high_watermark = 5%
+
+## The time interval for the periodic process limit check
+##
+## Value: Duration
+##
+## Default: 30s
+vm_mon.check_interval = 30s
+
+## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set.
+##
+## Default: 80%
+vm_mon.process_high_watermark = 80%
+
+## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear.
+##
+## Default: 60%
+vm_mon.process_low_watermark = 60%
+
 {{ additional_configs }}

include/emqx_client.hrl

@@ -12,15 +12,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 
--module(emqx_local_bridge_sup).
-
--include("emqx.hrl").
-
--export([start_link/3]).
-
--spec(start_link(node(), emqx_topic:topic(), [emqx_local_bridge:option()])
-      -> {ok, pid()} | {error, term()}).
-start_link(Node, Topic, Options) ->
-    MFA = {emqx_local_bridge, start_link, [Node, Topic, Options]},
-    emqx_pool_sup:start_link({bridge, Node, Topic}, random, MFA).
 
+-ifndef(EMQX_CLIENT_HRL).
+-define(EMQX_CLIENT_HRL, true).
+-record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false,
+                   packet_id, topic, props, payload}).
+-endif.

include/emqx_mqtt.hrl

@@ -171,9 +171,10 @@
 -define(RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED,   16#A2).
 
 %%--------------------------------------------------------------------
-%% Maximum MQTT Packet Length
+%% Maximum MQTT Packet ID and Length
 %%--------------------------------------------------------------------
 
+-define(MAX_PACKET_ID, 16#ffff).
 -define(MAX_PACKET_SIZE, 16#fffffff).
 
 %%--------------------------------------------------------------------

include/logger.hrl

@@ -35,6 +35,8 @@
 -define(ALERT(Format), ?LOG(alert, Format, [])).
 -define(ALERT(Format, Args), ?LOG(alert, Format, Args)).
 
+-define(LOG(Level, Format), ?LOG(Level, Format, [])).
+
 -define(LOG(Level, Format, Args),
         begin
           (logger:log(Level,#{},#{report_cb => fun(_) -> {(Format), (Args)} end}))

priv/emqx.schema

@@ -1512,22 +1512,6 @@ end}.
 %%--------------------------------------------------------------------
 %% Bridges
 %%--------------------------------------------------------------------
-{mapping, "bridge.$name.queue.mem_cache", "emqx.bridges", [
-  {datatype, flag}
-]}.
-
-{mapping, "bridge.$name.queue.batch_size", "emqx.bridges", [
-  {datatype, integer}
-]}.
-
-{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [
-  {datatype, string}
-]}.
-
-{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [
-  {datatype, bytesize}
-]}.
-
 {mapping, "bridge.$name.address", "emqx.bridges", [
   {datatype, string}
 ]}.
@@ -1616,11 +1600,27 @@ end}.
   {datatype, {duration, ms}}
 ]}.
 
-{mapping, "bridge.$name.max_inflight", "emqx.bridges", [
+{mapping, "bridge.$name.max_inflight_batches", "emqx.bridges", [
   {default, 0},
   {datatype, integer}
 ]}.
 
+{mapping, "bridge.$name.queue.batch_count_limit", "emqx.bridges", [
+  {datatype, integer}
+]}.
+
+{mapping, "bridge.$name.queue.batch_bytes_limit", "emqx.bridges", [
+  {datatype, bytesize}
+]}.
+
+{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [
+  {datatype, string}
+]}.
+
+{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [
+  {datatype, bytesize}
+]}.
+
 {translation, "emqx.bridges", fun(Conf) ->
     Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end,
 
@@ -1661,17 +1661,67 @@ end}.
                         lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, "subscription", I, "topic"], Topic} <- Configs])],
                                   [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, "subscription", I, "qos"], QoS} <- Configs])])
                     end,
-
+    IsNodeAddr = fun(Addr) ->
-    maps:to_list(
+                      case string:tokens(Addr, "@") of
-      lists:foldl(
+                          [_NodeName, _Hostname] -> true;
+                           _ -> false
+                      end
+                 end,
+    ConnMod = fun(Name) ->
+                      [AddrConfig] = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".address", Conf),
+                      {_, Addr} = AddrConfig,
+                      Subs = Subscriptions(Name),
+                      case IsNodeAddr(Addr) of
+                          true when Subs =/= [] ->
+                              error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs});
+                          true ->
+                              emqx_bridge_rpc;
+                          false ->
+                              emqx_bridge_mqtt
+                      end
+              end,
+    %% to be backward compatible
+    Translate =
+        fun Tr(queue, Q, Cfg) ->
+                NewQ = maps:fold(Tr, #{}, Q),
+                Cfg#{queue => NewQ};
+            Tr(address, Addr0, Cfg) ->
+                Addr = case IsNodeAddr(Addr0) of
+                           true -> list_to_atom(Addr0);
+                           false -> Addr0
+                       end,
+                Cfg#{address => Addr};
+            Tr(batch_size, Count, Cfg) ->
+                Cfg#{batch_count_limit => Count};
+            Tr(reconnect_interval, Ms, Cfg) ->
+                Cfg#{reconnect_delay_ms => Ms};
+            Tr(max_inflight, Count, Cfg) ->
+                Cfg#{max_inflight_batches => Count};
+            Tr(proto_ver, Ver, Cfg) ->
+                Cfg#{proto_ver =>
+                   case Ver of
+                       mqttv3 -> v3;
+                       mqttv4 -> v4;
+                       mqttv5 -> v5;
+                       _ -> v4
+                   end};
+            Tr(Key, Value, Cfg) ->
+                Cfg#{Key => Value}
+         end,
+    C = lists:foldl(
         fun({["bridge", Name, Opt], Val}, Acc) ->
                 %% e.g #{aws => [{OptKey, OptVal}]}
-                Init = [{list_to_atom(Opt), Val},{subscriptions, Subscriptions(Name)}, {queue, Queue(Name)}],
+                Init = [{list_to_atom(Opt), Val},
-                maps:update_with(list_to_atom(Name),
+                        {connect_module, ConnMod(Name)},
-                                 fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc);
+                        {subscriptions, Subscriptions(Name)},
+                        {queue, Queue(Name)}],
+                maps:update_with(list_to_atom(Name), fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc);
            (_, Acc) -> Acc
-        end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf))))
+        end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf))),
-
+    C1 = maps:map(fun(Bn, Bc) ->
+                      maps:to_list(maps:fold(Translate, #{}, maps:from_list(Bc)))
+                  end, C),
+    maps:to_list(C1)
 end}.
 
 %%--------------------------------------------------------------------
@@ -1844,3 +1894,70 @@ end}.
    {busy_port, cuttlefish:conf_get("sysmon.busy_port", Conf)},
    {busy_dist_port, cuttlefish:conf_get("sysmon.busy_dist_port", Conf)}]
 end}.
+
+%%--------------------------------------------------------------------
+%% Operating System Monitor
+%%--------------------------------------------------------------------
+
+{mapping, "os_mon.cpu_check_interval", "emqx.os_mon", [
+  {default, 60},
+  {datatype, {duration, s}}
+]}.
+
+{mapping, "os_mon.cpu_high_watermark", "emqx.os_mon", [
+  {default, "80%"},
+  {datatype, {percent, float}}
+]}.
+
+{mapping, "os_mon.cpu_low_watermark", "emqx.os_mon", [
+  {default, "60%"},
+  {datatype, {percent, float}}
+]}.
+
+{mapping, "os_mon.mem_check_interval", "emqx.os_mon", [
+  {default, 60},
+  {datatype, {duration, s}}
+]}.
+
+{mapping, "os_mon.sysmem_high_watermark", "emqx.os_mon", [
+  {default, "70%"},
+  {datatype, {percent, float}}
+]}.
+
+{mapping, "os_mon.procmem_high_watermark", "emqx.os_mon", [
+  {default, "5%"},
+  {datatype, {percent, float}}
+]}.
+
+{translation, "emqx.os_mon", fun(Conf) ->
+  [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)},
+   {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf)},
+   {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf)},
+   {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)},
+   {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf)},
+   {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf)}]
+end}.
+
+%%--------------------------------------------------------------------
+%% VM Monitor
+%%--------------------------------------------------------------------
+{mapping, "vm_mon.check_interval", "emqx.vm_mon", [
+  {default, 30},
+  {datatype, {duration, s}}
+]}.
+
+{mapping, "vm_mon.process_high_watermark", "emqx.vm_mon", [
+  {default, "80%"},
+  {datatype, {percent, float}}
+]}.
+
+{mapping, "vm_mon.process_low_watermark", "emqx.vm_mon", [
+  {default, "60%"},
+  {datatype, {percent, float}}
+]}.
+
+{translation, "emqx.vm_mon", fun(Conf) ->
+  [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)},
+   {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf)},
+   {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf)}]
+end}.

rebar.config

@@ -27,3 +27,5 @@
 {cover_export_enabled, true}.
 
 {plugins, [coveralls]}.
+
+{profiles, [{test, [{deps, [{meck, "0.8.13"}]}]}]}.

src/emqx.app.src

@@ -4,7 +4,7 @@
               {modules,[]},
               {registered,[emqx_sup]},
               {applications,[kernel,stdlib,jsx,gproc,gen_rpc,esockd,cowboy,
-                             replayq]},
+                             replayq,sasl,os_mon]},
               {env,[]},
               {mod,{emqx_app,[]}},
               {maintainers,["Feng Lee <feng@emqx.io>"]},

src/emqx_alarm_handler.erl

@@ -0,0 +1,169 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_alarm_handler).
+
+-behaviour(gen_event).
+
+-include("emqx.hrl").
+-include("logger.hrl").
+
+%% Mnesia bootstrap
+-export([mnesia/1]).
+
+-boot_mnesia({mnesia, [boot]}).
+-copy_mnesia({mnesia, [copy]}).
+
+-export([init/1,
+         handle_event/2,
+         handle_call/2,
+         handle_info/2,
+         terminate/2]).
+
+-export([load/0,
+         get_alarms/0]).
+
+-record(common_alarm, {id, desc}).
+-record(alarm_history, {id, clear_at}).
+
+-define(ALARM_TAB, emqx_alarm).
+-define(ALARM_HISTORY_TAB, emqx_alarm_history).
+
+%%------------------------------------------------------------------------------
+%% Mnesia bootstrap
+%%------------------------------------------------------------------------------
+
+mnesia(boot) ->
+    ok = ekka_mnesia:create_table(?ALARM_TAB, [
+                {type, set},
+                {disc_copies, [node()]},
+                {local_content, true},
+                {record_name, common_alarm},
+                {attributes, record_info(fields, common_alarm)}]),
+    ok = ekka_mnesia:create_table(?ALARM_HISTORY_TAB, [
+                {type, set},
+                {disc_copies, [node()]},
+                {local_content, true},
+                {record_name, alarm_history},
+                {attributes, record_info(fields, alarm_history)}]);
+mnesia(copy) ->
+    ok = ekka_mnesia:copy_table(?ALARM_TAB),
+    ok = ekka_mnesia:copy_table(?ALARM_HISTORY_TAB).
+
+%%----------------------------------------------------------------------
+%% API
+%%----------------------------------------------------------------------
+
+load() ->
+    gen_event:swap_handler(alarm_handler, {alarm_handler, swap}, {?MODULE, []}).
+
+get_alarms() ->
+    gen_event:call(alarm_handler, ?MODULE, get_alarms).
+
+%%----------------------------------------------------------------------
+%% gen_event callbacks
+%%----------------------------------------------------------------------
+
+init({_Args, {alarm_handler, ExistingAlarms}}) ->
+    init_tables(ExistingAlarms),
+    {ok, []};
+init(_) ->
+    init_tables([]),
+    {ok, []}.
+
+handle_event({set_alarm, {AlarmId, AlarmDesc = #alarm{timestamp = undefined}}}, State) ->
+    handle_event({set_alarm, {AlarmId, AlarmDesc#alarm{timestamp = os:timestamp()}}}, State);
+handle_event({set_alarm, Alarm = {AlarmId, AlarmDesc}}, State) ->
+    ?LOG(notice, "Alarm report: set ~p", [Alarm]),
+    case encode_alarm(Alarm) of
+        {ok, Json} ->
+            emqx_broker:safe_publish(alarm_msg(topic(alert, maybe_to_binary(AlarmId)), Json));
+        {error, Reason} ->
+            ?LOG(error, "Failed to encode alarm: ~p", [Reason])
+    end,
+    set_alarm_(AlarmId, AlarmDesc),
+    {ok, State};
+handle_event({clear_alarm, AlarmId}, State) ->
+    ?LOG(notice, "Alarm report: clear ~p", [AlarmId]),
+    emqx_broker:safe_publish(alarm_msg(topic(clear, maybe_to_binary(AlarmId)), <<"">>)),
+    clear_alarm_(AlarmId),
+    {ok, State};
+handle_event(_, State) ->
+    {ok, State}.
+
+handle_info(_, State) -> {ok, State}.
+
+handle_call(get_alarms, State) ->
+    {ok, get_alarms_(), State};
+handle_call(_Query, State)     -> {ok, {error, bad_query}, State}.
+
+terminate(swap, _State) ->
+    {emqx_alarm_handler, get_alarms_()};
+terminate(_, _) ->
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+init_tables(ExistingAlarms) ->
+    mnesia:clear_table(?ALARM_TAB),
+    lists:foreach(fun({Id, _Desc}) ->
+                      set_alarm_history(Id)
+                  end, ExistingAlarms).
+
+encode_alarm({AlarmId, #alarm{severity  = Severity, 
+                              title     = Title,
+                              summary   = Summary, 
+                              timestamp = Ts}}) ->
+    emqx_json:safe_encode([{id, maybe_to_binary(AlarmId)},
+                           {desc, [{severity, Severity},
+                                   {title, iolist_to_binary(Title)},
+                                   {summary, iolist_to_binary(Summary)},
+                                   {ts, emqx_time:now_secs(Ts)}]}]);
+encode_alarm({AlarmId, AlarmDesc}) ->
+    emqx_json:safe_encode([{id, maybe_to_binary(AlarmId)}, 
+                           {desc, maybe_to_binary(AlarmDesc)}]).
+
+alarm_msg(Topic, Payload) ->
+    Msg = emqx_message:make(?MODULE, Topic, Payload),
+    emqx_message:set_headers(#{'Content-Type' => <<"application/json">>},
+                             emqx_message:set_flag(sys, Msg)).
+
+topic(alert, AlarmId) ->
+    emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>);
+topic(clear, AlarmId) ->
+    emqx_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>).
+
+maybe_to_binary(Data) when is_binary(Data) ->
+    Data;
+maybe_to_binary(Data) ->
+    iolist_to_binary(io_lib:format("~p", [Data])).
+
+set_alarm_(Id, Desc) ->
+    mnesia:dirty_write(?ALARM_TAB, #common_alarm{id = Id, desc = Desc}).
+
+clear_alarm_(Id) ->
+    mnesia:dirty_delete(?ALARM_TAB, Id),
+    set_alarm_history(Id).
+
+get_alarms_() ->
+    Alarms = ets:tab2list(?ALARM_TAB),
+    [{Id, Desc} || #common_alarm{id = Id, desc = Desc} <- Alarms].
+
+set_alarm_history(Id) ->
+    mnesia:dirty_write(?ALARM_HISTORY_TAB, #alarm_history{id = Id,
+                                                          clear_at = undefined}).
+
+

src/emqx_alarm_mgr.erl

@@ -1,144 +0,0 @@
-%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-
--module(emqx_alarm_mgr).
-
--behaviour(gen_event).
-
--include("emqx.hrl").
--include("logger.hrl").
-
--export([start_link/0]).
--export([alarm_fun/0, get_alarms/0, set_alarm/1, clear_alarm/1]).
--export([add_alarm_handler/1, add_alarm_handler/2, delete_alarm_handler/1]).
-
-%% gen_event callbacks
--export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2,
-         code_change/3]).
-
--define(ALARM_MGR, ?MODULE).
-
-start_link() ->
-    start_with(
-      fun(Pid) ->
-          gen_event:add_handler(Pid, ?MODULE, [])
-      end).
-
-start_with(Fun) ->
-    case gen_event:start_link({local, ?ALARM_MGR}) of
-        {ok, Pid} -> Fun(Pid), {ok, Pid};
-        Error     -> Error
-    end.
-
-alarm_fun() -> alarm_fun(false).
-
-alarm_fun(Bool) ->
-    fun(alert, _Alarm)   when Bool =:= true  -> alarm_fun(true);
-       (alert,  Alarm)   when Bool =:= false -> set_alarm(Alarm), alarm_fun(true);
-       (clear,  AlarmId) when Bool =:= true  -> clear_alarm(AlarmId), alarm_fun(false);
-       (clear, _AlarmId) when Bool =:= false -> alarm_fun(false)
-    end.
-
--spec(set_alarm(emqx_types:alarm()) -> ok).
-set_alarm(Alarm) when is_record(Alarm, alarm) ->
-    gen_event:notify(?ALARM_MGR, {set_alarm, Alarm}).
-
--spec(clear_alarm(any()) -> ok).
-clear_alarm(AlarmId) when is_binary(AlarmId) ->
-    gen_event:notify(?ALARM_MGR, {clear_alarm, AlarmId}).
-
--spec(get_alarms() -> list(emqx_types:alarm())).
-get_alarms() ->
-    gen_event:call(?ALARM_MGR, ?MODULE, get_alarms).
-
-add_alarm_handler(Module) when is_atom(Module) ->
-    gen_event:add_handler(?ALARM_MGR, Module, []).
-
-add_alarm_handler(Module, Args) when is_atom(Module) ->
-    gen_event:add_handler(?ALARM_MGR, Module, Args).
-
-delete_alarm_handler(Module) when is_atom(Module) ->
-    gen_event:delete_handler(?ALARM_MGR, Module, []).
-
-%%------------------------------------------------------------------------------
-%% Default Alarm handler
-%%------------------------------------------------------------------------------
-
-init(_) -> {ok, #{alarms => []}}.
-
-handle_event({set_alarm, Alarm = #alarm{timestamp = undefined}}, State)->
-    handle_event({set_alarm, Alarm#alarm{timestamp = os:timestamp()}}, State);
-
-handle_event({set_alarm, Alarm = #alarm{id = AlarmId}}, State = #{alarms := Alarms}) ->
-    case encode_alarm(Alarm) of
-        {ok, Json} ->
-            emqx_broker:safe_publish(alarm_msg(alert, AlarmId, Json));
-        {error, Reason} ->
-            ?ERROR("[AlarmMgr] Failed to encode alarm: ~p", [Reason])
-    end,
-    {ok, State#{alarms := [Alarm|Alarms]}};
-
-handle_event({clear_alarm, AlarmId}, State = #{alarms := Alarms}) ->
-    case emqx_json:safe_encode([{id, AlarmId}, {ts, os:system_time(second)}]) of
-        {ok, Json} ->
-            emqx_broker:safe_publish(alarm_msg(clear, AlarmId, Json));
-        {error, Reason} ->
-            ?ERROR("[AlarmMgr] Failed to encode clear: ~p", [Reason])
-    end,
-    {ok, State#{alarms := lists:keydelete(AlarmId, 2, Alarms)}, hibernate};
-
-handle_event(Event, State)->
-    ?ERROR("[AlarmMgr] unexpected event: ~p", [Event]),
-    {ok, State}.
-
-handle_info(Info, State) ->
-    ?ERROR("[AlarmMgr] unexpected info: ~p", [Info]),
-    {ok, State}.
-
-handle_call(get_alarms, State = #{alarms := Alarms}) ->
-    {ok, Alarms, State};
-
-handle_call(Req, State) ->
-    ?ERROR("[AlarmMgr] unexpected call: ~p", [Req]),
-    {ok, ignored, State}.
-
-terminate(swap, State) ->
-    {?MODULE, State};
-terminate(_, _) ->
-    ok.
-
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-%%------------------------------------------------------------------------------
-%% Internal functions
-%%------------------------------------------------------------------------------
-
-encode_alarm(#alarm{id = AlarmId, severity = Severity, title = Title,
-                    summary = Summary, timestamp = Ts}) ->
-    emqx_json:safe_encode([{id, AlarmId}, {severity, Severity},
-                           {title, iolist_to_binary(Title)},
-                           {summary, iolist_to_binary(Summary)},
-                           {ts, emqx_time:now_secs(Ts)}]).
-
-alarm_msg(Type, AlarmId, Json) ->
-    Msg = emqx_message:make(?ALARM_MGR, topic(Type, AlarmId), Json),
-    emqx_message:set_headers( #{'Content-Type' => <<"application/json">>},
-                              emqx_message:set_flag(sys, Msg)).
-
-topic(alert, AlarmId) ->
-    emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>);
-topic(clear, AlarmId) ->
-    emqx_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>).
-

src/emqx_app.erl

@@ -40,6 +40,10 @@ start(_Type, _Args) ->
     emqx_listeners:start(),
     start_autocluster(),
     register(emqx, self()),
+
+    emqx_alarm_handler:load(),
+    emqx_logger_handler:init(),
+    
     print_vsn(),
     {ok, Sup}.
 

src/emqx_bridge.erl

@@ -1,4 +1,4 @@
-%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -12,452 +12,543 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 
+%% @doc Bridge works in two layers (1) batching layer (2) transport layer
+%% The `bridge' batching layer collects local messages in batches and sends over
+%% to remote MQTT node/cluster via `connetion' transport layer.
+%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be
+%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection'
+%% has to be `emqx_bridge_mqtt'.
+%%
+%% ```
+%% +------+                        +--------+
+%% | EMQX |                        | REMOTE |
+%% |      |                        |        |
+%% |   (bridge) <==(connection)==> |        |
+%% |      |                        |        |
+%% |      |                        |        |
+%% +------+                        +--------+
+%% '''
+%%
+%%
+%% This module implements 2 kinds of APIs with regards to batching and
+%% messaging protocol. (1) A `gen_statem' based local batch collector;
+%% (2) APIs for incoming remote batches/messages.
+%%
+%% Batch collector state diagram
+%%
+%% [standing_by] --(0) --> [connecting] --(2)--> [connected]
+%%                          |        ^                 |
+%%                          |        |                 |
+%%                          '--(1)---'--------(3)------'
+%%
+%% (0): auto or manual start
+%% (1): retry timeout
+%% (2): successfuly connected to remote node/cluster
+%% (3): received {disconnected, conn_ref(), Reason} OR
+%%      failed to send to remote node/cluster.
+%%
+%% NOTE: A bridge worker may subscribe to multiple (including wildcard)
+%% local topics, and the underlying `emqx_bridge_connect' may subscribe to
+%% multiple remote topics, however, worker/connections are not designed
+%% to support automatic load-balancing, i.e. in case it can not keep up
+%% with the amount of messages comming in, administrator should split and
+%% balance topics between worker/connections manually.
+%%
+%% NOTES:
+%% * Local messages are all normalised to QoS-1 when exporting to remote
+
 -module(emqx_bridge).
+-behaviour(gen_statem).
 
--behaviour(gen_server).
+%% APIs
+-export([start_link/2,
+         import_batch/2,
+         handle_ack/2,
+         stop/1]).
 
--include("emqx.hrl").
+%% gen_statem callbacks
+-export([terminate/3, code_change/4, init/1, callback_mode/0]).
+
+%% state functions
+-export([standing_by/3, connecting/3, connected/3]).
+
+%% management APIs
+-export([ensure_started/1, ensure_started/2, ensure_stopped/1, ensure_stopped/2, status/1]).
+-export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]).
+-export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]).
+
+-export_type([config/0,
+              batch/0,
+              ack_ref/0]).
+
+-type id() :: atom() | string() | pid().
+-type qos() :: emqx_mqtt_types:qos().
+-type config() :: map().
+-type batch() :: [emqx_bridge_msg:exp_msg()].
+-type ack_ref() :: term().
+-type topic() :: emqx_topic:topic().
+
+-include("logger.hrl").
 -include("emqx_mqtt.hrl").
 
--import(proplists, [get_value/2, get_value/3]).
+%% same as default in-flight limit for emqx_client
+-define(DEFAULT_BATCH_COUNT, 32).
+-define(DEFAULT_BATCH_BYTES, 1 bsl 20).
+-define(DEFAULT_SEND_AHEAD, 8).
+-define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)).
+-define(DEFAULT_SEG_BYTES, (1 bsl 20)).
+-define(maybe_send, {next_event, internal, maybe_send}).
 
--export([start_link/2, start_bridge/1, stop_bridge/1, status/1]).
+%% @doc Start a bridge worker. Supported configs:
+%% start_type: 'manual' (default) or 'auto', when manual, bridge will stay
+%%      at 'standing_by' state until a manual call to start it.
+%% connect_module: The module which implements emqx_bridge_connect behaviour
+%%      and work as message batch transport layer
+%% reconnect_delay_ms: Delay in milli-seconds for the bridge worker to retry
+%%      in case of transportation failure.
+%% max_inflight_batches: Max number of batches allowed to send-ahead before
+%%      receiving confirmation from remote node/cluster
+%% mountpoint: The topic mount point for messages sent to remote node/cluster
+%%      `undefined', `<<>>' or `""' to disable
+%% forwards: Local topics to subscribe.
+%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each
+%%      send call towards emqx_bridge_connect
+%% queue.batch_count_limit: Max number of messages to collect in a batch for
+%%      each send call towards emqx_bridge_connect
+%% queue.replayq_dir: Directory where replayq should persist messages
+%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file
+%%
+%% Find more connection specific configs in the callback modules
+%% of emqx_bridge_connect behaviour.
+start_link(Name, Config) when is_list(Config) ->
+    start_link(Name, maps:from_list(Config));
+start_link(Name, Config) ->
+    gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []).
 
--export([show_forwards/1, add_forward/2, del_forward/2]).
+%% @doc Manually start bridge worker. State idempotency ensured.
+ensure_started(Name) ->
+    gen_statem:call(name(Name), ensure_started).
 
--export([show_subscriptions/1, add_subscription/3, del_subscription/2]).
+ensure_started(Name, Config) ->
-
+    case start_link(Name, Config) of
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
+        {ok, Pid} -> {ok, Pid};
-         code_change/3]).
+        {error, {already_started,Pid}} -> {ok, Pid}
-
--record(state, {client_pid         :: pid(),
-                options            :: list(),
-                reconnect_interval :: pos_integer(),
-                mountpoint         :: binary(),
-                readq              :: list(),
-                writeq             :: list(),
-                replayq            :: map(),
-                ackref             :: replayq:ack_ref(),
-                queue_option       :: map(),
-                forwards           :: list(),
-                subscriptions      :: list()}).
-
--record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false,
-                   packet_id, topic, props, payload}).
-
-start_link(Name, Options) ->
-    gen_server:start_link({local, name(Name)}, ?MODULE, [Options], []).
-
-start_bridge(Name) ->
-    gen_server:call(name(Name), start_bridge).
-
-stop_bridge(Name) ->
-    gen_server:call(name(Name), stop_bridge).
-
--spec(show_forwards(atom()) -> list()).
-show_forwards(Name) ->
-    gen_server:call(name(Name), show_forwards).
-
--spec(add_forward(atom(), binary()) -> ok | {error, already_exists | validate_fail}).
-add_forward(Name, Topic) ->
-    try emqx_topic:validate({filter, Topic}) of
-        true ->
-            gen_server:call(name(Name), {add_forward, Topic})
-    catch
-        _Error:_Reason ->
-            {error, validate_fail}
     end.
 
--spec(del_forward(atom(), binary()) -> ok | {error, validate_fail}).
+%% @doc Manually stop bridge worker. State idempotency ensured.
-del_forward(Name, Topic) ->
+ensure_stopped(Id) ->
-    try emqx_topic:validate({filter, Topic}) of
+    ensure_stopped(Id, 1000).
-        true ->
+
-            gen_server:call(name(Name), {del_forward, Topic})
+ensure_stopped(Id, Timeout) ->
-    catch
+    Pid = case id(Id) of
-        _Error:_Reason ->
+              P when is_pid(P) -> P;
-            {error, validate_fail}
+              N -> whereis(N)
+          end,
+    case Pid of
+        undefined ->
+            ok;
+        _ ->
+            MRef = monitor(process, Pid),
+            unlink(Pid),
+            _ = gen_statem:call(id(Id), ensure_stopped, Timeout),
+            receive
+                {'DOWN', MRef, _, _, _} ->
+                    ok
+            after
+                Timeout ->
+                    exit(Pid, kill)
+            end
     end.
 
--spec(show_subscriptions(atom()) -> list()).
+stop(Pid) -> gen_statem:stop(Pid).
-show_subscriptions(Name) ->
-    gen_server:call(name(Name), show_subscriptions).
-
--spec(add_subscription(atom(), binary(), integer()) -> ok | {error, already_exists | validate_fail}).
-add_subscription(Name, Topic, QoS) ->
-    try emqx_topic:validate({filter, Topic}) of
-        true ->
-            gen_server:call(name(Name), {add_subscription, Topic, QoS})
-    catch
-        _Error:_Reason ->
-            {error, validate_fail}
-    end.
-
--spec(del_subscription(atom(), binary()) -> ok | {error, validate_fail}).
-del_subscription(Name, Topic) ->
-    try emqx_topic:validate({filter, Topic}) of
-        true ->
-            gen_server:call(name(Name), {del_subscription, Topic})
-    catch
-        error:_Reason ->
-            {error, validate_fail}
-    end.
 
 status(Pid) ->
-    gen_server:call(Pid, status).
+    gen_statem:call(Pid, status).
 
-%%------------------------------------------------------------------------------
+%% @doc This function is to be evaluated on message/batch receiver side.
-%% gen_server callbacks
+-spec import_batch(batch(), fun(() -> ok)) -> ok.
-%%------------------------------------------------------------------------------
+import_batch(Batch, AckFun) ->
+    lists:foreach(fun emqx_broker:publish/1, emqx_bridge_msg:to_broker_msgs(Batch)),
+    AckFun().
 
-init([Options]) ->
+%% @doc This function is to be evaluated on message/batch exporter side
-    process_flag(trap_exit, true),
+%% when message/batch is accepted by remote node.
-    case get_value(start_type, Options, manual) of
+-spec handle_ack(pid(), ack_ref()) -> ok.
-        manual -> ok;
+handle_ack(Pid, Ref) when node() =:= node(Pid) ->
-        auto -> erlang:send_after(1000, self(), start)
+    Pid ! {batch_ack, Ref},
-    end,
-    ReconnectInterval = get_value(reconnect_interval, Options, 30000),
-    Mountpoint = format_mountpoint(get_value(mountpoint, Options)),
-    QueueOptions = get_value(queue, Options),
-    {ok, #state{mountpoint         = Mountpoint,
-                queue_option       = QueueOptions,
-                readq              = [],
-                writeq             = [],
-                options            = Options,
-                reconnect_interval = ReconnectInterval}}.
-
-handle_call(start_bridge, _From, State = #state{client_pid = undefined}) ->
-    {Msg, NewState} = bridge(start, State),
-    {reply, #{msg => Msg}, NewState};
-
-handle_call(start_bridge, _From, State) ->
-    {reply, #{msg => <<"bridge already started">>}, State};
-
-handle_call(stop_bridge, _From, State = #state{client_pid = undefined}) ->
-    {reply, #{msg => <<"bridge not started">>}, State};
-
-handle_call(stop_bridge, _From, State = #state{client_pid = Pid}) ->
-    emqx_client:disconnect(Pid),
-    {reply, #{msg => <<"stop bridge successfully">>}, State};
-
-handle_call(status, _From, State = #state{client_pid = undefined}) ->
-    {reply, #{status => <<"Stopped">>}, State};
-handle_call(status, _From, State = #state{client_pid = _Pid})->
-    {reply, #{status => <<"Running">>}, State};
-
-handle_call(show_forwards, _From, State = #state{forwards = Forwards}) ->
-    {reply, Forwards, State};
-
-handle_call({add_forward, Topic}, _From, State = #state{forwards = Forwards}) ->
-    case not lists:member(Topic, Forwards) of
-        true ->
-            emqx_broker:subscribe(Topic),
-            {reply, ok, State#state{forwards = [Topic | Forwards]}};
-        false ->
-            {reply, {error, already_exists}, State}
-    end;
-
-handle_call({del_forward, Topic}, _From, State = #state{forwards = Forwards}) ->
-    case lists:member(Topic, Forwards) of
-        true ->
-            emqx_broker:unsubscribe(Topic),
-            {reply, ok, State#state{forwards = lists:delete(Topic, Forwards)}};
-        false ->
-            {reply, ok, State}
-    end;
-
-handle_call(show_subscriptions, _From, State = #state{subscriptions = Subscriptions}) ->
-    {reply, Subscriptions, State};
-
-handle_call({add_subscription, Topic, Qos}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) ->
-    case not lists:keymember(Topic, 1, Subscriptions) of
-        true ->
-            emqx_client:subscribe(ClientPid, {Topic, Qos}),
-            {reply, ok, State#state{subscriptions = [{Topic, Qos} | Subscriptions]}};
-        false ->
-            {reply, {error, already_exists}, State}
-    end;
-
-handle_call({del_subscription, Topic}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) ->
-    case lists:keymember(Topic, 1, Subscriptions) of
-        true ->
-            emqx_client:unsubscribe(ClientPid, Topic),
-            {reply, ok, State#state{subscriptions = lists:keydelete(Topic, 1, Subscriptions)}};
-        false ->
-            {reply, ok, State}
-    end;
-
-handle_call(Req, _From, State) ->
-    emqx_logger:error("[Bridge] unexpected call: ~p", [Req]),
-    {reply, ignored, State}.
-
-handle_cast(Msg, State) ->
-    emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]),
-    {noreply, State}.
-
-%%----------------------------------------------------------------
-%% Start or restart bridge
-%%----------------------------------------------------------------
-handle_info(start, State) ->
-    {_Msg, NewState} = bridge(start, State),
-    {noreply, NewState};
-
-handle_info(restart, State) ->
-    {_Msg, NewState} = bridge(restart, State),
-    {noreply, NewState};
-
-%%----------------------------------------------------------------
-%% pop message from replayq and publish again
-%%----------------------------------------------------------------
-handle_info(pop, State = #state{writeq = WriteQ, replayq = ReplayQ,
-                                queue_option = #{batch_size := BatchSize}}) ->
-    {NewReplayQ, AckRef, NewReadQ} = replayq:pop(ReplayQ, #{count_limit => BatchSize}),
-    {NewReadQ1, NewWriteQ} = case NewReadQ of
-                                 [] -> {WriteQ, []};
-                                 _ -> {NewReadQ, WriteQ}
-                             end,
-    self() ! replay,
-    {noreply, State#state{readq = NewReadQ1, writeq = NewWriteQ, replayq = NewReplayQ, ackref = AckRef}};
-
-handle_info(dump, State = #state{writeq = WriteQ, replayq = ReplayQ}) ->
-    NewReplayQueue = replayq:append(ReplayQ, lists:reverse(WriteQ)),
-    {noreply, State#state{replayq = NewReplayQueue, writeq = []}};
-
-%%----------------------------------------------------------------
-%% replay message from replayq
-%%----------------------------------------------------------------
-handle_info(replay, State = #state{client_pid = ClientPid, readq = ReadQ}) ->
-    {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []),
-    {noreply, State#state{readq = NewReadQ}};
-
-%%----------------------------------------------------------------
-%% received local node message
-%%----------------------------------------------------------------
-handle_info({dispatch, _, #message{topic = Topic, qos = QoS, payload = Payload, flags = #{retain := Retain}}},
-            State = #state{client_pid = undefined,
-                           mountpoint = Mountpoint})
-  when QoS =< 1 ->
-    Msg = #mqtt_msg{qos = 1,
-                    retain = Retain,
-                    topic = mountpoint(Mountpoint, Topic),
-                    payload = Payload},
-    {noreply, en_writeq({undefined, Msg}, State)};
-handle_info({dispatch, _, #message{topic = Topic, qos = QoS ,payload = Payload, flags = #{retain := Retain}}},
-            State = #state{client_pid = Pid,
-                           mountpoint = Mountpoint})
-  when QoS =< 1 ->
-    Msg = #mqtt_msg{qos     = 1,
-                    retain  = Retain,
-                    topic   = mountpoint(Mountpoint, Topic),
-                    payload = Payload},
-    case emqx_client:publish(Pid, Msg) of
-        {ok, PktId} ->
-            {noreply, en_writeq({PktId, Msg}, State)};
-        {error, {PktId, Reason}} ->
-            emqx_logger:error("[Bridge] Publish fail:~p", [Reason]),
-            {noreply, en_writeq({PktId, Msg}, State)}
-    end;
-
-%%----------------------------------------------------------------
-%% received remote node message
-%%----------------------------------------------------------------
-handle_info({publish, #{qos := QoS, dup := Dup, retain := Retain, topic := Topic,
-                        properties := Props, payload := Payload}}, State) ->
-    NewMsg0 = emqx_message:make(bridge, QoS, Topic, Payload),
-    NewMsg1 = emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, NewMsg0)),
-    emqx_broker:publish(NewMsg1),
-    {noreply, State};
-
-%%----------------------------------------------------------------
-%% received remote puback message
-%%----------------------------------------------------------------
-handle_info({puback, #{packet_id := PktId}}, State) ->
-    {noreply, delete(PktId, State)};
-
-handle_info({'EXIT', Pid, normal}, State = #state{client_pid = Pid}) ->
-    emqx_logger:warning("[Bridge] stop ~p", [normal]),
-    self() ! dump,
-    {noreply, State#state{client_pid = undefined}};
-
-handle_info({'EXIT', Pid, Reason}, State = #state{client_pid = Pid,
-                                                  reconnect_interval = ReconnectInterval}) ->
-    emqx_logger:error("[Bridge] stop ~p", [Reason]),
-    self() ! dump,
-    erlang:send_after(ReconnectInterval, self(), restart),
-    {noreply, State#state{client_pid = undefined}};
-
-handle_info(Info, State) ->
-    emqx_logger:error("[Bridge] unexpected info: ~p", [Info]),
-    {noreply, State}.
-
-terminate(_Reason, #state{}) ->
     ok.
 
-code_change(_OldVsn, State, _Extra) ->
+%% @doc Return all forwards (local subscriptions).
-    {ok, State}.
+-spec get_forwards(id()) -> [topic()].
+get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)).
 
-subscribe_remote_topics(ClientPid, Subscriptions) ->
+%% @doc Return all subscriptions (subscription over mqtt connection to remote broker).
-    [begin emqx_client:subscribe(ClientPid, {bin(Topic), Qos}), {bin(Topic), Qos} end
+-spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}].
-        || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})].
+get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions).
 
-subscribe_local_topics(Options) ->
+%% @doc Add a new forward (local topic subscription).
-    Topics = get_value(forwards, Options, []),
+-spec ensure_forward_present(id(), topic()) -> ok.
-    Subid = get_value(client_id, Options, <<"bridge">>),
+ensure_forward_present(Id, Topic) ->
-    [begin emqx_broker:subscribe(bin(Topic), #{qos => 1, subid => Subid}), bin(Topic) end
+    gen_statem:call(id(Id), {ensure_present, forwards, topic(Topic)}).
-        || Topic <- Topics, emqx_topic:validate({filter, bin(Topic)})].
 
-proto_ver(mqttv3) -> v3;
+%% @doc Ensure a forward topic is deleted.
-proto_ver(mqttv4) -> v4;
+-spec ensure_forward_absent(id(), topic()) -> ok.
-proto_ver(mqttv5) -> v5.
+ensure_forward_absent(Id, Topic) ->
-address(Address) ->
+    gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}).
-    case string:tokens(Address, ":") of
+
-        [Host] -> {Host, 1883};
+%% @doc Ensure subscribed to remote topic.
-        [Host, Port] -> {Host, list_to_integer(Port)}
+%% NOTE: only applicable when connection module is emqx_bridge_mqtt
+%%       return `{error, no_remote_subscription_support}' otherwise.
+-spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}.
+ensure_subscription_present(Id, Topic, QoS) ->
+    gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}).
+
+%% @doc Ensure unsubscribed from remote topic.
+%% NOTE: only applicable when connection module is emqx_bridge_mqtt
+-spec ensure_subscription_absent(id(), topic()) -> ok.
+ensure_subscription_absent(Id, Topic) ->
+    gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}).
+
+callback_mode() -> [state_functions, state_enter].
+
+%% @doc Config should be a map().
+init(Config) ->
+    erlang:process_flag(trap_exit, true),
+    Get = fun(K, D) -> maps:get(K, Config, D) end,
+    QCfg = maps:get(queue, Config, #{}),
+    GetQ = fun(K, D) -> maps:get(K, QCfg, D) end,
+    Dir = GetQ(replayq_dir, undefined),
+    QueueConfig =
+        case Dir =:= undefined orelse Dir =:= "" of
+            true -> #{mem_only => true};
+            false -> #{dir => Dir,
+                       seg_bytes => GetQ(replayq_seg_bytes, ?DEFAULT_SEG_BYTES)
+                      }
+        end,
+    Queue = replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1,
+                                      marshaller => fun msg_marshaller/1}),
+    Topics = lists:sort([iolist_to_binary(T) || T <- Get(forwards, [])]),
+    Subs = lists:keysort(1, lists:map(fun({T0, QoS}) ->
+                                              T = iolist_to_binary(T0),
+                                              true = emqx_topic:validate({filter, T}),
+                                              {T, QoS}
+                                      end, Get(subscriptions, []))),
+    ConnectModule = maps:get(connect_module, Config),
+    ConnectConfig = maps:without([connect_module,
+                                  queue,
+                                  reconnect_delay_ms,
+                                  max_inflight_batches,
+                                  mountpoint,
+                                  forwards
+                                 ], Config#{subscriptions => Subs}),
+    ConnectFun = fun(SubsX) -> emqx_bridge_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end,
+    {ok, standing_by,
+     #{connect_module => ConnectModule,
+       connect_fun => ConnectFun,
+       start_type => Get(start_type, manual),
+       reconnect_delay_ms => maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS),
+       batch_bytes_limit => GetQ(batch_bytes_limit, ?DEFAULT_BATCH_BYTES),
+       batch_count_limit => GetQ(batch_count_limit, ?DEFAULT_BATCH_COUNT),
+       max_inflight_batches => Get(max_inflight_batches, ?DEFAULT_SEND_AHEAD),
+       mountpoint => format_mountpoint(Get(mountpoint, undefined)),
+       forwards => Topics,
+       subscriptions => Subs,
+       replayq => Queue,
+       inflight => []
+      }}.
+
+code_change(_Vsn, State, Data, _Extra) ->
+    {ok, State, Data}.
+
+terminate(_Reason, _StateName, #{replayq := Q} = State) ->
+    _ = disconnect(State),
+    _ = replayq:close(Q),
+    ok.
+
+%% @doc Standing by for manual start.
+standing_by(enter, _, #{start_type := auto}) ->
+    Action = {state_timeout, 0, do_connect},
+    {keep_state_and_data, Action};
+standing_by(enter, _, #{start_type := manual}) ->
+    keep_state_and_data;
+standing_by({call, From}, ensure_started, State) ->
+    {next_state, connecting, State,
+     [{reply, From, ok}]};
+standing_by(state_timeout, do_connect, State) ->
+    {next_state, connecting, State};
+standing_by({call, From}, _Call, _State) ->
+    {keep_state_and_data, [{reply, From, {error,standing_by}}]};
+standing_by(info, Info, State) ->
+    ?LOG(info, "Bridge ~p discarded info event at state standing_by:\n~p", [name(), Info]),
+    {keep_state_and_data, State};
+standing_by(Type, Content, State) ->
+    common(standing_by, Type, Content, State).
+
+%% @doc Connecting state is a state with timeout.
+%% After each timeout, it re-enters this state and start a retry until
+%% successfuly connected to remote node/cluster.
+connecting(enter, connected, #{reconnect_delay_ms := Timeout}) ->
+    Action = {state_timeout, Timeout, reconnect},
+    {keep_state_and_data, Action};
+connecting(enter, _, #{reconnect_delay_ms := Timeout,
+                       connect_fun := ConnectFun,
+                       subscriptions := Subs,
+                       forwards := Forwards
+                      } = State) ->
+    ok = subscribe_local_topics(Forwards),
+    case ConnectFun(Subs) of
+        {ok, ConnRef, Conn} ->
+            ?LOG(info, "Bridge ~p connected", [name()]),
+            Action = {state_timeout, 0, connected},
+            {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action};
+        error ->
+            Action = {state_timeout, Timeout, reconnect},
+            {keep_state_and_data, Action}
+    end;
+connecting(state_timeout, connected, State) ->
+    {next_state, connected, State};
+connecting(state_timeout, reconnect, _State) ->
+    repeat_state_and_data;
+connecting(info, {batch_ack, Ref}, State) ->
+    case do_ack(State, Ref) of
+        {ok, NewState} ->
+            {keep_state, NewState};
+        _ ->
+            keep_state_and_data
+    end;
+connecting(internal, maybe_send, _State) ->
+    keep_state_and_data;
+connecting(info, {disconnected, _Ref, _Reason}, _State) ->
+    keep_state_and_data;
+connecting(Type, Content, State) ->
+    common(connecting, Type, Content, State).
+
+%% @doc Send batches to remote node/cluster when in 'connected' state.
+connected(enter, _OldState, #{inflight := Inflight} = State) ->
+    case retry_inflight(State#{inflight := []}, Inflight) of
+        {ok, NewState} ->
+            Action = {state_timeout, 0, success},
+            {keep_state, NewState, Action};
+        {error, NewState} ->
+            Action = {state_timeout, 0, failure},
+            {keep_state, disconnect(NewState), Action}
+    end;
+connected(state_timeout, failure, State) ->
+    {next_state, connecting, State};
+connected(state_timeout, success, State) ->
+    {keep_state, State, ?maybe_send};
+connected(internal, maybe_send, State) ->
+    case pop_and_send(State) of
+        {ok, NewState} ->
+            {keep_state, NewState};
+        {error, NewState} ->
+            {next_state, connecting, disconnect(NewState)}
+    end;
+connected(info, {disconnected, ConnRef, Reason},
+          #{conn_ref := ConnRefCurrent, connection := Conn} = State) ->
+    case ConnRefCurrent =:= ConnRef of
+        true ->
+            ?LOG(info, "Bridge ~p diconnected~nreason=~p", [name(), Conn, Reason]),
+            {next_state, connecting,
+             State#{conn_ref := undefined, connection := undefined}};
+        false ->
+            keep_state_and_data
+    end;
+connected(info, {batch_ack, Ref}, State) ->
+    case do_ack(State, Ref) of
+        stale ->
+            keep_state_and_data;
+        bad_order ->
+            %% try re-connect then re-send
+            ?LOG(error, "Bad order ack received by bridge ~p", [name()]),
+            {next_state, connecting, disconnect(State)};
+        {ok, NewState} ->
+            {keep_state, NewState, ?maybe_send}
+    end;
+connected(Type, Content, State) ->
+    common(connected, Type, Content, State).
+
+%% Common handlers
+common(StateName, {call, From}, status, _State) ->
+    {keep_state_and_data, [{reply, From, StateName}]};
+common(_StateName, {call, From}, ensure_started, _State) ->
+    {keep_state_and_data, [{reply, From, ok}]};
+common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) ->
+    {keep_state_and_data, [{reply, From, Forwards}]};
+common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) ->
+    {keep_state_and_data, [{reply, From, Subs}]};
+common(_StateName, {call, From}, {ensure_present, What, Topic}, State) ->
+    {Result, NewState} = ensure_present(What, Topic, State),
+    {keep_state, NewState, [{reply, From, Result}]};
+common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) ->
+    {Result, NewState} = ensure_absent(What, Topic, State),
+    {keep_state, NewState, [{reply, From, Result}]};
+common(_StateName, {call, From}, ensure_stopped, _State) ->
+    {stop_and_reply, {shutdown, manual},
+     [{reply, From, ok}]};
+common(_StateName, info, {dispatch, _, Msg},
+       #{replayq := Q} = State) ->
+    NewQ = replayq:append(Q, collect([Msg])),
+    {keep_state, State#{replayq => NewQ}, ?maybe_send};
+common(StateName, Type, Content, State) ->
+    ?LOG(info, "Bridge ~p discarded ~p type event at state ~p:\n~p",
+          [name(), Type, StateName, Content]),
+    {keep_state, State}.
+
+ensure_present(Key, Topic, State) ->
+    Topics = maps:get(Key, State),
+    case is_topic_present(Topic, Topics) of
+        true ->
+            {ok, State};
+        false ->
+            R = do_ensure_present(Key, Topic, State),
+            {R, State#{Key := lists:usort([Topic | Topics])}}
     end.
-options(Options) ->
-    options(Options, []).
-options([], Acc) ->
-    Acc;
-options([{username, Username}| Options], Acc) ->
-    options(Options, [{username, Username}|Acc]);
-options([{proto_ver, ProtoVer}| Options], Acc) ->
-    options(Options, [{proto_ver, proto_ver(ProtoVer)}|Acc]);
-options([{password, Password}| Options], Acc) ->
-    options(Options, [{password, Password}|Acc]);
-options([{keepalive, Keepalive}| Options], Acc) ->
-    options(Options, [{keepalive, Keepalive}|Acc]);
-options([{client_id, ClientId}| Options], Acc) ->
-    options(Options, [{client_id, ClientId}|Acc]);
-options([{clean_start, CleanStart}| Options], Acc) ->
-    options(Options, [{clean_start, CleanStart}|Acc]);
-options([{address, Address}| Options], Acc) ->
-    {Host, Port} = address(Address),
-    options(Options, [{host, Host}, {port, Port}|Acc]);
-options([{ssl, Ssl}| Options], Acc) ->
-    options(Options, [{ssl, Ssl}|Acc]);
-options([{ssl_opts, SslOpts}| Options], Acc) ->
-    options(Options, [{ssl_opts, SslOpts}|Acc]);
-options([_Option | Options], Acc) ->
-    options(Options, Acc).
 
-name(Id) ->
+ensure_absent(Key, Topic, State) ->
-    list_to_atom(lists:concat([?MODULE, "_", Id])).
+    Topics = maps:get(Key, State),
+    case is_topic_present(Topic, Topics) of
+        true ->
+            R = do_ensure_absent(Key, Topic, State),
+            {R, State#{Key := ensure_topic_absent(Topic, Topics)}};
+        false ->
+            {ok, State}
+    end.
 
-bin(L) -> iolist_to_binary(L).
+ensure_topic_absent(_Topic, []) -> [];
+ensure_topic_absent(Topic, [{_, _} | _] = L) -> lists:keydelete(Topic, 1, L);
+ensure_topic_absent(Topic, L) -> lists:delete(Topic, L).
 
-mountpoint(undefined, Topic) ->
+is_topic_present({Topic, _QoS}, Topics) ->
-    Topic;
+    is_topic_present(Topic, Topics);
-mountpoint(Prefix, Topic) ->
+is_topic_present(Topic, Topics) ->
-    <<Prefix/binary, Topic/binary>>.
+    lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics).
+
+do_ensure_present(forwards, Topic, _) ->
+    ok = subscribe_local_topic(Topic);
+do_ensure_present(subscriptions, {Topic, QoS},
+                  #{connect_module := ConnectModule, connection := Conn}) ->
+    case erlang:function_exported(ConnectModule, ensure_subscribed, 3) of
+        true ->
+            _ = ConnectModule:ensure_subscribed(Conn, Topic, QoS),
+            ok;
+        false ->
+            {error, no_remote_subscription_support}
+    end.
+
+do_ensure_absent(forwards, Topic, _) ->
+    ok = emqx_broker:unsubscribe(Topic);
+do_ensure_absent(subscriptions, Topic, #{connect_module := ConnectModule,
+                                         connection := Conn}) ->
+    case erlang:function_exported(ConnectModule, ensure_unsubscribed, 2) of
+        true -> ConnectModule:ensure_unsubscribed(Conn, Topic);
+        false -> {error, no_remote_subscription_support}
+    end.
+
+collect(Acc) ->
+    receive
+        {dispatch, _, Msg} ->
+            collect([Msg | Acc])
+    after
+        0 ->
+            lists:reverse(Acc)
+    end.
+
+%% Retry all inflight (previously sent but not acked) batches.
+retry_inflight(State, []) -> {ok, State};
+retry_inflight(#{inflight := Inflight} = State,
+               [#{q_ack_ref := QAckRef, batch := Batch} | T] = Remain) ->
+    case do_send(State, QAckRef, Batch) of
+        {ok, NewState} ->
+            retry_inflight(NewState, T);
+        {error, Reason} ->
+            ?LOG(error, "Inflight retry failed\n~p", [Reason]),
+            {error, State#{inflight := Inflight ++ Remain}}
+    end.
+
+pop_and_send(#{inflight := Inflight,
+               max_inflight_batches := Max
+              } = State) when length(Inflight) >= Max ->
+    {ok, State};
+pop_and_send(#{replayq := Q,
+               batch_count_limit := CountLimit,
+               batch_bytes_limit := BytesLimit
+              } = State) ->
+    case replayq:is_empty(Q) of
+        true ->
+            {ok, State};
+        false ->
+            Opts = #{count_limit => CountLimit, bytes_limit => BytesLimit},
+            {Q1, QAckRef, Batch} = replayq:pop(Q, Opts),
+            do_send(State#{replayq := Q1}, QAckRef, Batch)
+    end.
+
+%% Assert non-empty batch because we have a is_empty check earlier.
+do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) ->
+    case maybe_send(State, Batch) of
+        {ok, Ref} ->
+            %% this is a list of inflight BATCHes, not expecting it to be too long
+            NewInflight = Inflight ++ [#{q_ack_ref => QAckRef,
+                                         send_ack_ref => Ref,
+                                         batch => Batch}],
+            {ok, State#{inflight := NewInflight}};
+        {error, Reason} ->
+            ?LOG(info, "Batch produce failed\n~p", [Reason]),
+            {error, State}
+    end.
+
+do_ack(State = #{inflight := [#{send_ack_ref := Refx, q_ack_ref := QAckRef} | Rest],
+                 replayq := Q}, Ref) when Refx =:= Ref ->
+    ok = replayq:ack(Q, QAckRef),
+    {ok, State#{inflight := Rest}};
+do_ack(#{inflight := Inflight}, Ref) ->
+    case lists:any(fun(#{send_ack_ref := Ref0}) -> Ref0 =:= Ref end, Inflight) of
+        true -> bad_order;
+        false -> stale
+    end.
+
+subscribe_local_topics(Topics) -> lists:foreach(fun subscribe_local_topic/1, Topics).
+
+subscribe_local_topic(Topic0) ->
+    Topic = topic(Topic0),
+    try
+        emqx_topic:validate({filter, Topic})
+    catch
+        error : Reason ->
+            erlang:error({bad_topic, Topic, Reason})
+    end,
+    ok = emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}).
+
+topic(T) -> iolist_to_binary(T).
+
+disconnect(#{connection := Conn,
+             conn_ref := ConnRef,
+             connect_module := Module
+            } = State) when Conn =/= undefined ->
+    ok = Module:stop(ConnRef, Conn),
+    State#{conn_ref => undefined,
+           connection => undefined};
+disconnect(State) -> State.
+
+%% Called only when replayq needs to dump it to disk.
+msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin);
+msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg).
+
+%% Return {ok, SendAckRef} or {error, Reason}
+maybe_send(#{connect_module := Module,
+             connection := Connection,
+             mountpoint := Mountpoint
+            }, Batch) ->
+    Module:send(Connection, [emqx_bridge_msg:to_export(Module, Mountpoint, M) || M <- Batch]).
 
 format_mountpoint(undefined) ->
     undefined;
 format_mountpoint(Prefix) ->
-    binary:replace(bin(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)).
+    binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)).
 
-en_writeq(Msg, State = #state{replayq = ReplayQ,
+name() -> {_, Name} = process_info(self(), registered_name), Name.
-                              queue_option = #{mem_cache := false}}) ->
-    NewReplayQ = replayq:append(ReplayQ, [Msg]),
-    State#state{replayq = NewReplayQ};
-en_writeq(Msg, State = #state{writeq = WriteQ,
-                              queue_option = #{batch_size := BatchSize,
-                                               mem_cache := true}})
-  when length(WriteQ) < BatchSize->
-    State#state{writeq = [Msg | WriteQ]} ;
-en_writeq(Msg, State = #state{writeq = WriteQ, replayq = ReplayQ,
-                              queue_option = #{mem_cache := true}}) ->
-    NewReplayQ =replayq:append(ReplayQ, lists:reverse(WriteQ)),
-    State#state{writeq = [Msg], replayq = NewReplayQ}.
 
-publish_readq_msg(_ClientPid, [], NewReadQ) ->
+name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])).
-    {ok, NewReadQ};
-publish_readq_msg(ClientPid, [{_PktId, Msg} | ReadQ], NewReadQ) ->
-    {ok, PktId} = emqx_client:publish(ClientPid, Msg),
-    publish_readq_msg(ClientPid, ReadQ, [{PktId, Msg} | NewReadQ]).
 
-delete(PktId, State = #state{ replayq = ReplayQ,
+id(Pid) when is_pid(Pid) -> Pid;
-                              readq = [],
+id(Name) -> name(Name).
-                              queue_option = #{ mem_cache := false}}) ->
-    {NewReplayQ, NewAckRef, Msgs} = replayq:pop(ReplayQ, #{count_limit => 1}),
-    logger:debug("[Msg] PacketId ~p, Msg: ~p", [PktId, Msgs]),
-    ok = replayq:ack(NewReplayQ, NewAckRef),
-    case Msgs of
-        [{PktId, _Msg}] ->
-            self() ! pop,
-            State#state{ replayq = NewReplayQ, ackref = NewAckRef };
-        [{_PktId, _Msg}] ->
-            NewReplayQ1 = replayq:append(NewReplayQ, Msgs),
-            self() ! pop,
-            State#state{ replayq = NewReplayQ1, ackref = NewAckRef };
-        _Empty ->
-            State#state{ replayq = NewReplayQ, ackref = NewAckRef}
-    end;
-delete(_PktId, State = #state{readq = [], writeq = [], replayq = ReplayQ, ackref = AckRef}) ->
-    ok = replayq:ack(ReplayQ, AckRef),
-    self() ! pop,
-    State;
-
-delete(PktId, State = #state{readq = [], writeq = WriteQ}) ->
-    State#state{writeq = lists:keydelete(PktId, 1, WriteQ)};
-
-delete(PktId, State = #state{readq = ReadQ, replayq = ReplayQ, ackref = AckRef}) ->
-    NewReadQ = lists:keydelete(PktId, 1, ReadQ),
-    case NewReadQ of
-        [] ->
-            ok = replayq:ack(ReplayQ, AckRef),
-            self() ! pop;
-        _NewReadQ ->
-            ok
-    end,
-    State#state{ readq = NewReadQ }.
-
-bridge(Action, State = #state{options = Options,
-                              replayq = ReplayQ,
-                              queue_option
-                              = QueueOption
-                              = #{batch_size := BatchSize}})
-  when BatchSize > 0 ->
-    case emqx_client:start_link([{owner, self()} | options(Options)]) of
-        {ok, ClientPid} ->
-            case emqx_client:connect(ClientPid) of
-                {ok, _} ->
-                    emqx_logger:info("[Bridge] connected to remote successfully"),
-                    Subs = subscribe_remote_topics(ClientPid, get_value(subscriptions, Options, [])),
-                    Forwards = subscribe_local_topics(Options),
-                    {NewReplayQ, AckRef, ReadQ} = open_replayq(ReplayQ, QueueOption),
-                    {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []),
-                    {<<"start bridge successfully">>,
-                     State#state{client_pid = ClientPid,
-                                 subscriptions = Subs,
-                                 readq = NewReadQ,
-                                 replayq = NewReplayQ,
-                                 ackref = AckRef,
-                                 forwards = Forwards}};
-                {error, Reason} ->
-                    emqx_logger:error("[Bridge] connect to remote failed! error: ~p", [Reason]),
-                    {<<"connect to remote failed">>,
-                     State#state{client_pid = ClientPid}}
-            end;
-        {error, Reason} ->
-            emqx_logger:error("[Bridge] ~p failed! error: ~p", [Action, Reason]),
-            {<<"start bridge failed">>, State}
-    end;
-bridge(Action, State) ->
-    emqx_logger:error("[Bridge] ~p failed! error: batch_size should greater than zero", [Action]),
-    {<<"Open Replayq failed">>, State}.
-
-open_replayq(undefined, #{batch_size := BatchSize,
-                          replayq_dir := ReplayqDir,
-                          replayq_seg_bytes := ReplayqSegBytes}) ->
-    ReplayQ = replayq:open(#{dir => ReplayqDir,
-                             seg_bytes => ReplayqSegBytes,
-                             sizer => fun(Term) ->
-                                          size(term_to_binary(Term))
-                                      end,
-                             marshaller => fun({PktId, Msg}) ->
-                                               term_to_binary({PktId, Msg});
-                                              (Bin) ->
-                                               binary_to_term(Bin)
-                                           end}),
-    replayq:pop(ReplayQ, #{count_limit => BatchSize});
-open_replayq(ReplayQ, #{batch_size := BatchSize}) ->
-    replayq:pop(ReplayQ, #{count_limit => BatchSize}).

src/emqx_bridge_connect.erl

@@ -0,0 +1,71 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_bridge_connect).
+
+-export([start/2]).
+
+-export_type([config/0, connection/0]).
+
+-optional_callbacks([ensure_subscribed/3, ensure_unsubscribed/2]).
+
+%% map fields depend on implementation
+-type config() :: map().
+-type connection() :: term().
+-type conn_ref() :: term().
+-type batch() :: emqx_protal:batch().
+-type ack_ref() :: emqx_bridge:ack_ref().
+-type topic() :: emqx_topic:topic().
+-type qos() :: emqx_mqtt_types:qos().
+
+-include("logger.hrl").
+
+%% establish the connection to remote node/cluster
+%% protal worker (the caller process) should be expecting
+%% a message {disconnected, conn_ref()} when disconnected.
+-callback start(config()) -> {ok, conn_ref(), connection()} | {error, any()}.
+
+%% send to remote node/cluster
+%% bridge worker (the caller process) should be expecting
+%% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster
+-callback send(connection(), batch()) -> {ok, ack_ref()} | {error, any()}.
+
+%% called when owner is shutting down.
+-callback stop(conn_ref(), connection()) -> ok.
+
+-callback ensure_subscribed(connection(), topic(), qos()) -> ok.
+
+-callback ensure_unsubscribed(connection(), topic()) -> ok.
+
+start(Module, Config) ->
+    case Module:start(Config) of
+        {ok, Ref, Conn} ->
+            {ok, Ref, Conn};
+        {error, Reason} ->
+            Config1 = obfuscate(Config),
+            ?LOG(error, "Failed to connect with module=~p\n"
+                 "config=~p\nreason:~p", [Module, Config1, Reason]),
+            error
+    end.
+
+obfuscate(Map) ->
+    maps:fold(fun(K, V, Acc) ->
+                      case is_sensitive(K) of
+                          true -> [{K, '***'} | Acc];
+                          false -> [{K, V} | Acc]
+                      end
+              end, [], Map).
+
+is_sensitive(password) -> true;
+is_sensitive(_) -> false.

src/emqx_bridge_mqtt.erl

@@ -0,0 +1,185 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol
+
+-module(emqx_bridge_mqtt).
+-behaviour(emqx_bridge_connect).
+
+%% behaviour callbacks
+-export([start/1,
+         send/2,
+         stop/2
+        ]).
+
+%% optional behaviour callbacks
+-export([ensure_subscribed/3,
+         ensure_unsubscribed/2
+        ]).
+
+-include("emqx_mqtt.hrl").
+
+-define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}).
+
+%% Messages towards ack collector process
+-define(RANGE(Min, Max), {Min, Max}).
+-define(REF_IDS(Ref, Ids), {Ref, Ids}).
+-define(SENT(RefIds), {sent, RefIds}).
+-define(ACKED(AnyPktId), {acked, AnyPktId}).
+-define(STOP(Ref), {stop, Ref}).
+
+start(Config = #{address := Address}) ->
+    Ref = make_ref(),
+    Parent = self(),
+    AckCollector = spawn_link(fun() -> ack_collector(Parent, Ref) end),
+    Handlers = make_hdlr(Parent, AckCollector, Ref),
+    {Host, Port} = case string:tokens(Address, ":") of
+                       [H] -> {H, 1883};
+                       [H, P] -> {H, list_to_integer(P)}
+                   end,
+    ClientConfig = Config#{msg_handler => Handlers,
+                           owner => AckCollector,
+                           host => Host,
+                           port => Port},
+    case emqx_client:start_link(ClientConfig) of
+        {ok, Pid} ->
+            case emqx_client:connect(Pid) of
+                {ok, _} ->
+                    try
+                        subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])),
+                        {ok, Ref, #{ack_collector => AckCollector,
+                                    client_pid => Pid}}
+                    catch
+                        throw : Reason ->
+                            ok = stop(AckCollector, Pid),
+                            {error, Reason}
+                    end;
+                {error, Reason} ->
+                    ok = stop(Ref, #{ack_collector => AckCollector, client_pid => Pid}),
+                    {error, Reason}
+            end;
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+stop(Ref, #{ack_collector := AckCollector, client_pid := Pid}) ->
+    safe_stop(Pid, fun() -> emqx_client:stop(Pid) end, 1000),
+    safe_stop(AckCollector, fun() -> AckCollector ! ?STOP(Ref) end, 1000),
+    ok.
+
+ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) ->
+    emqx_client:subscribe(Pid, Topic, QoS);
+ensure_subscribed(_Conn, _Topic, _QoS) ->
+    %% return ok for now, next re-connect should should call start with new topic added to config
+    ok.
+
+ensure_unsubscribed(#{client_pid := Pid}, Topic) when is_pid(Pid) ->
+    emqx_client:unsubscribe(Pid, Topic);
+ensure_unsubscribed(_, _) ->
+    %% return ok for now, next re-connect should should call start with this topic deleted from config
+    ok.
+
+safe_stop(Pid, StopF, Timeout) ->
+    MRef = monitor(process, Pid),
+    unlink(Pid),
+    try
+        StopF()
+    catch
+        _ : _ ->
+            ok
+    end,
+    receive
+        {'DOWN', MRef, _, _, _} ->
+            ok
+    after
+        Timeout ->
+            exit(Pid, kill)
+    end.
+
+send(Conn, Batch) ->
+    send(Conn, Batch, []).
+
+send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, [Msg | Rest], Acc) ->
+    case emqx_client:publish(ClientPid, Msg) of
+        {ok, PktId} when Rest =:= [] ->
+            %% last one sent
+            Ref = make_ref(),
+            AckCollector ! ?SENT(?REF_IDS(Ref, lists:reverse([PktId | Acc]))),
+            {ok, Ref};
+        {ok, PktId} ->
+            send(Conn, Rest, [PktId | Acc]);
+        {error, Reason} ->
+            %% NOTE: There is no partial sucess of a batch and recover from the middle
+            %% only to retry all messages in one batch
+            {error, Reason}
+    end.
+
+ack_collector(Parent, ConnRef) ->
+    ack_collector(Parent, ConnRef, queue:new(), []).
+
+ack_collector(Parent, ConnRef, Acked, Sent) ->
+    {NewAcked, NewSent} =
+        receive
+            ?STOP(ConnRef) ->
+                exit(normal);
+            ?ACKED(PktId) ->
+                match_acks(Parent, queue:in(PktId, Acked), Sent);
+            ?SENT(RefIds) ->
+                %% this message only happens per-batch, hence ++ is ok
+                match_acks(Parent, Acked, Sent ++ [RefIds])
+        after
+            200 ->
+                {Acked, Sent}
+        end,
+   ack_collector(Parent, ConnRef, NewAcked, NewSent).
+
+match_acks(_Parent, Acked, []) -> {Acked, []};
+match_acks(Parent, Acked, Sent) ->
+    match_acks_1(Parent, queue:out(Acked), Sent).
+
+match_acks_1(_Parent, {empty, Empty}, Sent) -> {Empty, Sent};
+match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId]) | Sent]) ->
+    %% batch finished
+    ok = emqx_bridge:handle_ack(Parent, Ref),
+    match_acks(Parent, Acked, Sent);
+match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId | RestIds]) | Sent]) ->
+    %% one message finished, but not the whole batch
+    match_acks(Parent, Acked, [?REF_IDS(Ref, RestIds) | Sent]).
+
+
+%% When puback for QoS-1 message is received from remote MQTT broker
+%% NOTE: no support for QoS-2
+handle_puback(AckCollector, #{packet_id := PktId, reason_code := RC}) ->
+    RC =:= ?RC_SUCCESS orelse error({puback_error_code, RC}),
+    AckCollector ! ?ACKED(PktId),
+    ok.
+
+%% Message published from remote broker. Import to local broker.
+import_msg(Msg) ->
+    %% auto-ack should be enabled in emqx_client, hence dummy ack-fun.
+    emqx_bridge:import_batch([Msg], _AckFun = fun() -> ok end).
+
+make_hdlr(Parent, AckCollector, Ref) ->
+    #{puback => fun(Ack) -> handle_puback(AckCollector, Ack) end,
+      publish => fun(Msg) -> import_msg(Msg) end,
+      disconnected => fun(Reason) -> Parent ! {disconnected, Ref, Reason}, ok end
+     }.
+
+subscribe_remote_topics(ClientPid, Subscriptions) ->
+    lists:foreach(fun({Topic, Qos}) ->
+                          case emqx_client:subscribe(ClientPid, Topic, Qos) of
+                              {ok, _, _} -> ok;
+                              Error -> throw(Error)
+                          end
+                  end, Subscriptions).

src/emqx_bridge_msg.erl

@@ -0,0 +1,84 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_bridge_msg).
+
+-export([ to_binary/1
+        , from_binary/1
+        , to_export/3
+        , to_broker_msgs/1
+        , estimate_size/1
+        ]).
+
+-export_type([msg/0]).
+
+-include("emqx.hrl").
+-include("emqx_mqtt.hrl").
+-include("emqx_client.hrl").
+
+-type msg() :: emqx_types:message().
+-type exp_msg() :: emqx_types:message() | #mqtt_msg{}.
+
+%% @doc Make export format:
+%% 1. Mount topic to a prefix
+%% 2. Fix QoS to 1
+%% @end
+%% Shame that we have to know the callback module here
+%% would be great if we can get rid of #mqtt_msg{} record
+%% and use #message{} in all places.
+-spec to_export(emqx_bridge_rpc | emqx_bridge_mqtt,
+                undefined | binary(), msg()) -> exp_msg().
+to_export(emqx_bridge_mqtt, Mountpoint,
+          #message{topic = Topic,
+                   payload = Payload,
+                   flags = Flags
+                  }) ->
+    Retain = maps:get(retain, Flags, false),
+    #mqtt_msg{qos = ?QOS_1,
+              retain = Retain,
+              topic = topic(Mountpoint, Topic),
+              payload = Payload};
+to_export(_Module, Mountpoint,
+          #message{topic = Topic} = Msg) ->
+    Msg#message{topic = topic(Mountpoint, Topic), qos = 1}.
+
+%% @doc Make `binary()' in order to make iodata to be persisted on disk.
+-spec to_binary(msg()) -> binary().
+to_binary(Msg) -> term_to_binary(Msg).
+
+%% @doc Unmarshal binary into `msg()'.
+-spec from_binary(binary()) -> msg().
+from_binary(Bin) -> binary_to_term(Bin).
+
+%% @doc Estimate the size of a message.
+%% Count only the topic length + payload size
+-spec estimate_size(msg()) -> integer().
+estimate_size(#message{topic = Topic, payload = Payload}) ->
+    size(Topic) + size(Payload).
+
+%% @doc By message/batch receiver, transform received batch into
+%% messages to dispatch to local brokers.
+to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch).
+
+to_broker_msg(#message{} = Msg) ->
+    %% internal format from another EMQX node via rpc
+    Msg;
+to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic,
+                properties := Props, payload := Payload}) ->
+    %% published from remote node over a MQTT connection
+    emqx_message:set_headers(Props,
+        emqx_message:set_flags(#{dup => Dup, retain => Retain},
+            emqx_message:make(bridge, QoS, Topic, Payload))).
+
+topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic).

src/emqx_bridge_rpc.erl

@@ -0,0 +1,105 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+%% @doc This module implements EMQX Bridge transport layer based on gen_rpc.
+
+-module(emqx_bridge_rpc).
+-behaviour(emqx_bridge_connect).
+
+%% behaviour callbacks
+-export([start/1,
+         send/2,
+         stop/2
+        ]).
+
+%% Internal exports
+-export([ handle_send/2
+        , handle_ack/2
+        , heartbeat/2
+        ]).
+
+-type ack_ref() :: emqx_bridge:ack_ref().
+-type batch() :: emqx_bridge:batch().
+
+-define(HEARTBEAT_INTERVAL, timer:seconds(1)).
+
+-define(RPC, gen_rpc).
+
+start(#{address := Remote}) ->
+    case poke(Remote) of
+        ok ->
+            Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), Remote]),
+            {ok, Pid, Remote};
+        Error ->
+            Error
+    end.
+
+stop(Pid, _Remote) when is_pid(Pid) ->
+    Ref = erlang:monitor(process, Pid),
+    unlink(Pid),
+    Pid ! stop,
+    receive
+        {'DOWN', Ref, process, Pid, _Reason} ->
+            ok
+    after
+        1000 ->
+            exit(Pid, kill)
+    end,
+    ok.
+
+%% @doc Callback for `emqx_bridge_connect' behaviour
+-spec send(node(), batch()) -> {ok, ack_ref()} | {error, any()}.
+send(Remote, Batch) ->
+    Sender = self(),
+    case ?RPC:call(Remote, ?MODULE, handle_send, [Sender, Batch]) of
+        {ok, Ref} -> {ok, Ref};
+        {badrpc, Reason} -> {error, Reason}
+    end.
+
+%% @doc Handle send on receiver side.
+-spec handle_send(pid(), batch()) -> {ok, ack_ref()} | {error, any()}.
+handle_send(SenderPid, Batch) ->
+    SenderNode = node(SenderPid),
+    Ref = make_ref(),
+    AckFun = fun() -> ?RPC:cast(SenderNode, ?MODULE, handle_ack, [SenderPid, Ref]), ok end,
+    case emqx_bridge:import_batch(Batch, AckFun) of
+        ok -> {ok, Ref};
+        Error -> Error
+    end.
+
+%% @doc Handle batch ack in sender node.
+handle_ack(SenderPid, Ref) ->
+    ok = emqx_bridge:handle_ack(SenderPid, Ref).
+
+%% @hidden Heartbeat loop
+heartbeat(Parent, RemoteNode) ->
+    Interval = ?HEARTBEAT_INTERVAL,
+    receive
+        stop -> exit(normal)
+    after
+        Interval ->
+            case poke(RemoteNode) of
+                ok ->
+                    ?MODULE:heartbeat(Parent, RemoteNode);
+                {error, Reason} ->
+                    Parent ! {disconnected, self(), Reason},
+                    exit(normal)
+            end
+    end.
+
+poke(Node) ->
+    case ?RPC:call(Node, erlang, node, []) of
+        Node -> ok;
+        {badrpc, Reason} -> {error, Reason}
+    end.

src/emqx_bridge_sup.erl

@@ -13,33 +13,50 @@
 %% limitations under the License.
 
 -module(emqx_bridge_sup).
-
 -behavior(supervisor).
 
--include("emqx.hrl").
+-include("logger.hrl").
 
--export([start_link/0, bridges/0]).
+-export([start_link/0, start_link/1, bridges/0]).
-
+-export([create_bridge/2, drop_bridge/1]).
-%% Supervisor callbacks
 -export([init/1]).
 
-start_link() ->
+-define(SUP, ?MODULE).
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+-define(WORKER_SUP, emqx_bridge_worker_sup).
+
+start_link() -> start_link(?SUP).
+
+start_link(Name) ->
+    supervisor:start_link({local, Name}, ?MODULE, Name).
+
+init(?SUP) ->
+    BridgesConf = emqx_config:get_env(bridges, []),
+    BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf),
+    SupFlag = #{strategy => one_for_one,
+                intensity => 100,
+                period => 10},
+    {ok, {SupFlag, BridgeSpec}}.
+
+bridge_spec({Name, Config}) ->
+    #{id => Name,
+      start => {emqx_bridge, start_link, [Name, Config]},
+      restart => permanent,
+      shutdown => 5000,
+      type => worker,
+      modules => [emqx_bridge]}.
 
-%% @doc List all bridges
 -spec(bridges() -> [{node(), map()}]).
 bridges() ->
-    [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?MODULE)].
+    [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)].
 
-init([]) ->
+create_bridge(Id, Config) ->
-    BridgesOpts = emqx_config:get_env(bridges, []),
+    supervisor:start_child(?SUP, bridge_spec({Id, Config})).
-    Bridges = [spec(Opts)|| Opts <- BridgesOpts],
-    {ok, {{one_for_one, 10, 100}, Bridges}}.
 
-spec({Id, Options})->
+drop_bridge(Id) ->
-    #{id       => Id,
+    case supervisor:terminate_child(?SUP, Id) of
-      start    => {emqx_bridge, start_link, [Id, Options]},
+        ok ->
-      restart  => permanent,
+            supervisor:delete_child(?SUP, Id);
-      shutdown => 5000,
+        Error ->
-      type     => worker,
+            ?LOG(error, "[Bridge] Delete bridge failed", [Error]),
-      modules  => [emqx_bridge]}.
+            Error
+    end.

src/emqx_client.erl

@@ -18,6 +18,7 @@
 
 -include("types.hrl").
 -include("emqx_mqtt.hrl").
+-include("emqx_client.hrl").
 
 -export([start_link/0, start_link/1]).
 -export([request/5, request/6, request_async/7, receive_response/3]).
@@ -37,12 +38,12 @@
 %% For test cases
 -export([pause/1, resume/1]).
 
--export([initialized/3, waiting_for_connack/3, connected/3]).
+-export([initialized/3, waiting_for_connack/3, connected/3, inflight_full/3]).
 -export([init/1, callback_mode/0, handle_event/4, terminate/3, code_change/4]).
 
 -export_type([client/0, properties/0, payload/0, pubopt/0, subopt/0,
               request_input/0, response_payload/0, request_handler/0,
-              corr_data/0]).
+              corr_data/0, mqtt_msg/0]).
 
 -export_type([host/0, option/0]).
 
@@ -58,7 +59,7 @@
 
 -define(RESPONSE_TIMEOUT_SECONDS, timer:seconds(5)).
 
--define(NO_HANDLER, undefined).
+-define(NO_REQ_HANDLER, undefined).
 
 -define(NO_GROUP, <<>>).
 
@@ -66,10 +67,23 @@
 
 -type(host() :: inet:ip_address() | inet:hostname()).
 
--type corr_data() :: binary().
+-type(corr_data() :: binary()).
+
+%% NOTE: Message handler is different from request handler.
+%% Message handler is a set of callbacks defined to handle MQTT messages as well as
+%% the disconnect event.
+%% Request handler is a callback to handle received MQTT message as in 'request',
+%% and publish another MQTT message back to the defined topic as in 'response'.
+%% `owner' and `msg_handler' has no effect when `request_handler' is set.
+-define(NO_MSG_HDLR, undefined).
+-type(msg_handler() :: #{puback := fun((_) -> any()),
+                         publish := fun((emqx_types:message()) -> any()),
+                         disconnected := fun(({reason_code(), _Properties :: term()}) -> any())
+                        }).
 
 -type(option() :: {name, atom()}
                 | {owner, pid()}
+                | {msg_handler, msg_handler()}
                 | {host, host()}
                 | {hosts, [{host(), inet:port_number()}]}
                 | {port, inet:port_number()}
@@ -97,13 +111,11 @@
                 | {force_ping, boolean()}
                 | {properties, properties()}).
 
--record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false,
-                   packet_id, topic, props, payload}).
-
 -type(mqtt_msg() :: #mqtt_msg{}).
 
 -record(state, {name            :: atom(),
                 owner           :: pid(),
+                msg_handler     :: ?NO_MSG_HDLR | msg_handler(),
                 host            :: host(),
                 port            :: inet:port_number(),
                 hosts           :: [{host(), inet:port_number()}],
@@ -378,7 +390,7 @@ publish(Client, Topic, Properties, Payload, Opts)
                               payload = iolist_to_binary(Payload)}).
 
 -spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}).
-publish(Client, Msg) when is_record(Msg, mqtt_msg) ->
+publish(Client, Msg) ->
     gen_statem:call(Client, {publish, Msg}).
 
 -spec(unsubscribe(client(), topic() | [topic()]) -> subscribe_ret()).
@@ -499,7 +511,7 @@ init([Options]) ->
                                  auto_ack        = true,
                                  ack_timeout     = ?DEFAULT_ACK_TIMEOUT,
                                  retry_interval  = 0,
-                                 request_handler = ?NO_HANDLER,
+                                 request_handler = ?NO_REQ_HANDLER,
                                  connect_timeout = ?DEFAULT_CONNECT_TIMEOUT,
                                  last_packet_id  = 1}),
     {ok, initialized, init_parse_state(State)}.
@@ -518,6 +530,8 @@ init([{name, Name} | Opts], State) ->
 init([{owner, Owner} | Opts], State) when is_pid(Owner) ->
     link(Owner),
     init(Opts, State#state{owner = Owner});
+init([{msg_handler, Hdlr} | Opts], State) ->
+    init(Opts, State#state{msg_handler = Hdlr});
 init([{host, Host} | Opts], State) ->
     init(Opts, State#state{host = Host});
 init([{port, Port} | Opts], State) ->
@@ -729,12 +743,12 @@ waiting_for_connack(EventType, EventContent, State) ->
         false -> {stop, connack_timeout}
     end.
 
-connected({call, From}, subscriptions, State = #state{subscriptions = Subscriptions}) ->
+connected({call, From}, subscriptions, #state{subscriptions = Subscriptions}) ->
-    {keep_state, State, [{reply, From, maps:to_list(Subscriptions)}]};
+    {keep_state_and_data, [{reply, From, maps:to_list(Subscriptions)}]};
 
 connected({call, From}, info, State) ->
     Info = lists:zip(record_info(fields, state), tl(tuple_to_list(State))),
-    {keep_state, State, [{reply, From, Info}]};
+    {keep_state_and_data, [{reply, From, Info}]};
 
 connected({call, From}, pause, State) ->
     {keep_state, State#state{paused = true}, [{reply, From, ok}]};
@@ -742,14 +756,11 @@ connected({call, From}, pause, State) ->
 connected({call, From}, resume, State) ->
     {keep_state, State#state{paused = false}, [{reply, From, ok}]};
 
-connected({call, From}, stop, _State) ->
+connected({call, From}, get_properties, #state{properties = Properties}) ->
-    {stop_and_reply, normal, [{reply, From, ok}]};
+    {keep_state_and_data, [{reply, From, Properties}]};
 
-connected({call, From}, get_properties, State = #state{properties = Properties}) ->
+connected({call, From}, client_id, #state{client_id = ClientId}) ->
-    {keep_state, State, [{reply, From, Properties}]};
+    {keep_state_and_data, [{reply, From, ClientId}]};
-
-connected({call, From}, client_id, State = #state{client_id = ClientId}) ->
-    {keep_state, State, [{reply, From, ClientId}]};
 
 connected({call, From}, {set_request_handler, RequestHandler}, State) ->
     {keep_state, State#state{request_handler = RequestHandler}, [{reply, From, ok}]};
@@ -779,19 +790,18 @@ connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) ->
 connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}},
           State = #state{inflight = Inflight, last_packet_id = PacketId})
     when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) ->
-    case emqx_inflight:is_full(Inflight) of
+    Msg1 = Msg#mqtt_msg{packet_id = PacketId},
-        true ->
+    case send(Msg1, State) of
-            {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]};
+        {ok, NewState} ->
-        false ->
+            Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight),
-            Msg1 = Msg#mqtt_msg{packet_id = PacketId},
+            State1 = ensure_retry_timer(NewState#state{inflight = Inflight1}),
-            case send(Msg1, State) of
+            Actions = [{reply, From, {ok, PacketId}}],
-                {ok, NewState} ->
+            case emqx_inflight:is_full(Inflight1) of
-                    Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight),
+                true -> {next_state, inflight_full, State1, Actions};
-                    {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}),
+                false -> {keep_state, State1, Actions}
-                     [{reply, From, {ok, PacketId}}]};
+            end;
-                {error, Reason} ->
+        {error, Reason} ->
-                    {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]}
+            {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]}
-            end
     end;
 
 connected({call, From}, UnsubReq = {unsubscribe, Properties, Topics},
@@ -833,8 +843,8 @@ connected(cast, {pubrel, PacketId, ReasonCode, Properties}, State) ->
 connected(cast, {pubcomp, PacketId, ReasonCode, Properties}, State) ->
     send_puback(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), State);
 
-connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), State = #state{paused = true}) ->
+connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), #state{paused = true}) ->
-    {keep_state, State};
+    keep_state_and_data;
 
 connected(cast, Packet = ?PUBLISH_PACKET(?QOS_0, _Topic, _PacketId, Properties, Payload),
           State) when Properties =/= undefined ->
@@ -858,18 +868,8 @@ connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _Topic, _PacketId, Properties,
 connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) ->
     publish_process(?QOS_2, Packet, State);
 
-connected(cast, ?PUBACK_PACKET(PacketId, ReasonCode, Properties),
+connected(cast, ?PUBACK_PACKET(_PacketId, _ReasonCode, _Properties) = PubAck, State) ->
-          State = #state{owner = Owner, inflight = Inflight}) ->
+    {keep_state, delete_inflight(PubAck, State)};
-    case emqx_inflight:lookup(PacketId, Inflight) of
-        {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} ->
-            Owner ! {puback, #{packet_id   => PacketId,
-                               reason_code => ReasonCode,
-                               properties  => Properties}},
-            {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}};
-        none ->
-            emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]),
-            {keep_state, State}
-    end;
 
 connected(cast, ?PUBREC_PACKET(PacketId), State = #state{inflight = Inflight}) ->
     send_puback(?PUBREL_PACKET(PacketId),
@@ -897,21 +897,11 @@ connected(cast, ?PUBREL_PACKET(PacketId),
              end;
          error ->
              emqx_logger:warning("Unexpected PUBREL: ~p", [PacketId]),
-             {keep_state, State}
+             keep_state_and_data
      end;
 
-connected(cast, ?PUBCOMP_PACKET(PacketId, ReasonCode, Properties),
+connected(cast, ?PUBCOMP_PACKET(_PacketId, _ReasonCode, _Properties) = PubComp, State) ->
-          State = #state{owner = Owner, inflight = Inflight}) ->
+    {keep_state, delete_inflight(PubComp, State)};
-    case emqx_inflight:lookup(PacketId, Inflight) of
-        {value, {pubrel, _PacketId, _Ts}} ->
-            Owner ! {puback, #{packet_id   => PacketId,
-                               reason_code => ReasonCode,
-                               properties  => Properties}},
-            {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}};
-        none ->
-            emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]),
-            {keep_state, State}
-     end;
 
 connected(cast, ?SUBACK_PACKET(PacketId, Properties, ReasonCodes),
           State = #state{subscriptions = _Subscriptions}) ->
@@ -920,7 +910,8 @@ connected(cast, ?SUBACK_PACKET(PacketId, Properties, ReasonCodes),
             %%TODO: Merge reason codes to subscriptions?
             Reply = {ok, Properties, ReasonCodes},
             {keep_state, NewState, [{reply, From, Reply}]};
-        false -> {keep_state, State}
+        false ->
+            keep_state_and_data
     end;
 
 connected(cast, ?UNSUBACK_PACKET(PacketId, Properties, ReasonCodes),
@@ -933,22 +924,22 @@ connected(cast, ?UNSUBACK_PACKET(PacketId, Properties, ReasonCodes),
                           end, Subscriptions, Topics),
             {keep_state, NewState#state{subscriptions = Subscriptions1},
              [{reply, From, {ok, Properties, ReasonCodes}}]};
-        false -> {keep_state, State}
+        false ->
+            keep_state_and_data
     end;
 
-connected(cast, ?PACKET(?PINGRESP), State = #state{pending_calls = []}) ->
+connected(cast, ?PACKET(?PINGRESP), #state{pending_calls = []}) ->
-    {keep_state, State};
+    keep_state_and_data;
 connected(cast, ?PACKET(?PINGRESP), State) ->
     case take_call(ping, State) of
         {value, #call{from = From}, NewState} ->
             {keep_state, NewState, [{reply, From, pong}]};
-        false -> {keep_state, State}
+        false ->
+            keep_state_and_data
     end;
 
-connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties),
+connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), State) ->
-          State = #state{owner = Owner}) ->
+    {stop, {disconnected, ReasonCode, Properties}, State};
-    Owner ! {disconnected, ReasonCode, Properties},
-    {stop, disconnected, State};
 
 connected(info, {timeout, _TRef, keepalive}, State = #state{force_ping = true}) ->
     case send(?PACKET(?PINGREQ), State) of
@@ -989,15 +980,19 @@ connected(info, {timeout, TRef, retry}, State = #state{retry_timer = TRef,
 connected(EventType, EventContent, Data) ->
     handle_event(EventType, EventContent, connected, Data).
 
-should_ping(Sock) ->
+inflight_full({call, _From}, {publish, #mqtt_msg{qos = QoS}}, _State) when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) ->
-    case emqx_client_sock:getstat(Sock, [send_oct]) of
+    {keep_state_and_data, [postpone]};
-        {ok, [{send_oct, Val}]} ->
+inflight_full(cast, ?PUBACK_PACKET(_PacketId, _ReasonCode, _Properties) = PubAck, State) ->
-            OldVal = get(send_oct), put(send_oct, Val),
+    delete_inflight_when_full(PubAck, State);
-            OldVal == undefined orelse OldVal == Val;
+inflight_full(cast, ?PUBCOMP_PACKET(_PacketId, _ReasonCode, _Properties) = PubComp, State) ->
-        Error = {error, _Reason} ->
+    delete_inflight_when_full(PubComp, State);
-            Error
+inflight_full(EventType, EventContent, Data) ->
-    end.
+    %% inflight_full is a sub-state of connected state,
+    %% delegate all other events to connected state.
+    connected(EventType, EventContent, Data).
 
+handle_event({call, From}, stop, _StateName, _State) ->
+    {stop_and_reply, normal, [{reply, From, ok}]};
 handle_event(info, {TcpOrSsL, _Sock, Data}, _StateName, State)
     when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl ->
     emqx_logger:debug("RECV Data: ~p", [Data]),
@@ -1017,23 +1012,31 @@ handle_event(info, {'EXIT', Owner, Reason}, _, State = #state{owner = Owner}) ->
     emqx_logger:debug("[~p] Got EXIT from owner, Reason: ~p", [?MODULE, Reason]),
     {stop, {shutdown, Reason}, State};
 
-handle_event(info, {inet_reply, _Sock, ok}, _, State) ->
+handle_event(info, {inet_reply, _Sock, ok}, _, _State) ->
-    {keep_state, State};
+    keep_state_and_data;
 
 handle_event(info, {inet_reply, _Sock, {error, Reason}}, _, State) ->
     emqx_logger:error("[~p] got tcp error: ~p", [?MODULE, Reason]),
     {stop, {shutdown, Reason}, State};
 
-handle_event(EventType, EventContent, StateName, StateData) ->
+handle_event(EventType, EventContent, StateName, _StateData) ->
     emqx_logger:error("State: ~s, Unexpected Event: (~p, ~p)",
                       [StateName, EventType, EventContent]),
-    {keep_state, StateData}.
+    keep_state_and_data.
 
 %% Mandatory callback functions
-terminate(_Reason, _State, #state{socket = undefined}) ->
+terminate(Reason, _StateName, State = #state{socket = Socket}) ->
-    ok;
+    case Reason of
-terminate(_Reason, _State, #state{socket = Socket}) ->
+        {disconnected, ReasonCode, Properties} ->
-    emqx_client_sock:close(Socket).
+            %% backward compatible
+            ok = eval_msg_handler(State, disconnected, {ReasonCode, Properties});
+        _ ->
+            ok = eval_msg_handler(State, disconnected, Reason)
+    end,
+    case Socket =:= undefined of
+        true -> ok;
+        _ -> emqx_client_sock:close(Socket)
+    end.
 
 code_change(_Vsn, State, Data, _Extra) ->
     {ok, State, Data}.
@@ -1042,6 +1045,47 @@ code_change(_Vsn, State, Data, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
+should_ping(Sock) ->
+    case emqx_client_sock:getstat(Sock, [send_oct]) of
+        {ok, [{send_oct, Val}]} ->
+            OldVal = get(send_oct), put(send_oct, Val),
+            OldVal == undefined orelse OldVal == Val;
+        Error = {error, _Reason} ->
+            Error
+    end.
+
+delete_inflight(?PUBACK_PACKET(PacketId, ReasonCode, Properties),
+                State = #state{inflight = Inflight}) ->
+    case emqx_inflight:lookup(PacketId, Inflight) of
+        {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} ->
+            ok = eval_msg_handler(State, puback, #{packet_id   => PacketId,
+                                                   reason_code => ReasonCode,
+                                                   properties  => Properties}),
+            State#state{inflight = emqx_inflight:delete(PacketId, Inflight)};
+        none ->
+            emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]),
+            State
+    end;
+delete_inflight(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties),
+                State = #state{inflight = Inflight}) ->
+    case emqx_inflight:lookup(PacketId, Inflight) of
+        {value, {pubrel, _PacketId, _Ts}} ->
+            ok = eval_msg_handler(State, puback, #{packet_id   => PacketId,
+                                                   reason_code => ReasonCode,
+                                                   properties  => Properties}),
+            State#state{inflight = emqx_inflight:delete(PacketId, Inflight)};
+        none ->
+            emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]),
+            State
+     end.
+
+delete_inflight_when_full(Packet, State0) ->
+    State = #state{inflight = Inflight} = delete_inflight(Packet, State0),
+    case emqx_inflight:is_full(Inflight) of
+        true -> {keep_state, State};
+        false -> {next_state, connected, State}
+    end.
+
 %% Subscribe to response topic.
 -spec(sub_response_topic(client(), qos(), topic()) -> ok).
 sub_response_topic(Client, QoS, Topic) when is_binary(Topic) ->
@@ -1103,8 +1147,8 @@ assign_id(?NO_CLIENT_ID, Props) ->
 assign_id(Id, _Props) ->
     Id.
 
-publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State = #state{auto_ack = AutoAck}) ->
+publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State0 = #state{auto_ack = AutoAck}) ->
-    _ = deliver(packet_to_msg(Packet), State),
+    State = deliver(packet_to_msg(Packet), State0),
     case AutoAck of
         true  -> send_puback(?PUBACK_PACKET(PacketId), State);
         false -> {keep_state, State}
@@ -1118,18 +1162,11 @@ publish_process(?QOS_2, Packet = ?PUBLISH_PACKET(?QOS_2, PacketId),
         Stop -> Stop
     end.
 
-response_publish(undefined, State, _QoS, _Payload) ->
+response_publish(#{'Response-Topic' := ResponseTopic} = Properties,
-    State;
+                 State = #state{request_handler = RequestHandler}, QoS, Payload)
-response_publish(Properties, State = #state{request_handler = RequestHandler}, QoS, Payload) ->
+      when RequestHandler =/= ?NO_REQ_HANDLER ->
-    case maps:find('Response-Topic', Properties) of
+    do_publish(ResponseTopic, Properties, State, QoS, Payload);
-        {ok, ResponseTopic} ->
+response_publish(_Properties, State, _QoS, _Payload) -> State.
-            case RequestHandler of
-                ?NO_HANDLER -> State;
-                _ -> do_publish(ResponseTopic, Properties, State, QoS, Payload)
-            end;
-        _ ->
-            State
-    end.
 
 do_publish(ResponseTopic, Properties, State = #state{request_handler = RequestHandler}, ?QOS_0, Payload) ->
         Msg = #mqtt_msg{qos     = ?QOS_0,
@@ -1210,11 +1247,12 @@ ensure_ack_timer(State = #state{ack_timer     = undefined,
 ensure_ack_timer(State) -> State.
 
 ensure_retry_timer(State = #state{retry_interval = Interval}) ->
-    ensure_retry_timer(Interval, State).
+    do_ensure_retry_timer(Interval, State).
-ensure_retry_timer(Interval, State = #state{retry_timer = undefined})
+
+do_ensure_retry_timer(Interval, State = #state{retry_timer = undefined})
     when Interval > 0 ->
     State#state{retry_timer = erlang:start_timer(Interval, self(), retry)};
-ensure_retry_timer(_Interval, State) ->
+do_ensure_retry_timer(_Interval, State) ->
     State.
 
 retry_send(State = #state{inflight = Inflight}) ->
@@ -1231,7 +1269,7 @@ retry_send([{Type, Msg, Ts} | Msgs], Now, State = #state{retry_interval = Interv
                      {ok, NewState} -> retry_send(Msgs, Now, NewState);
                      {error, Error} -> {stop, Error}
                  end;
-        false -> {keep_state, ensure_retry_timer(Interval - Diff, State)}
+        false -> {keep_state, do_ensure_retry_timer(Interval - Diff, State)}
     end.
 
 retry_send(publish, Msg = #mqtt_msg{qos = QoS, packet_id = PacketId},
@@ -1253,19 +1291,37 @@ retry_send(pubrel, PacketId, Now, State = #state{inflight = Inflight}) ->
             Error
     end.
 
+deliver(_Msg, State = #state{request_handler = Hdlr}) when Hdlr =/= ?NO_REQ_HANDLER ->
+    %% message has been terminated by request handler, hence should not continue processing
+    State;
 deliver(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId,
                   topic = Topic, props = Props, payload = Payload},
-        State = #state{owner = Owner, request_handler = RequestHandler}) ->
+        State) ->
-    case RequestHandler of
+    Msg = #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId,
-        ?NO_HANDLER ->
+            topic => Topic, properties => Props, payload => Payload,
-            Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId,
+            client_pid => self()},
-                                topic => Topic, properties => Props, payload => Payload,
+    ok = eval_msg_handler(State, publish, Msg),
-                                client_pid => self()}};
-        _ ->
-            ok
-    end,
     State.
 
+eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER,
+                        owner = Owner},
+                 disconnected, {ReasonCode, Properties}) ->
+    %% Special handling for disconnected message when there is no handler callback
+    Owner ! {disconnected, ReasonCode, Properties},
+    ok;
+eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER},
+                 disconnected, _OtherReason) ->
+    %% do nothing to be backward compatible
+    ok;
+eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER,
+                        owner = Owner}, Kind, Msg) ->
+    Owner ! {Kind, Msg},
+    ok;
+eval_msg_handler(#state{msg_handler = Handler}, Kind, Msg) ->
+    F = maps:get(Kind, Handler),
+    _ = F(Msg),
+    ok.
+
 packet_to_msg(#mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
                                                           dup    = Dup,
                                                           qos    = QoS,
@@ -1319,9 +1375,9 @@ send(Msg, State) when is_record(Msg, mqtt_msg) ->
 send(Packet, State = #state{socket = Sock, proto_ver = Ver})
     when is_record(Packet, mqtt_packet) ->
     Data = emqx_frame:serialize(Packet, #{version => Ver}),
-    emqx_logger:debug("SEND Data: ~p", [Data]),
+    emqx_logger:debug("SEND Data: ~1000p", [Packet]),
     case emqx_client_sock:send(Sock, Data) of
-        ok  -> {ok, next_packet_id(State)};
+        ok  -> {ok, bump_last_packet_id(State)};
         Error -> Error
     end.
 
@@ -1355,10 +1411,11 @@ next_events(Packets) ->
     [{next_event, cast, Packet} || Packet <- lists:reverse(Packets)].
 
 %%------------------------------------------------------------------------------
-%% Next packet id
+%% packet_id generation
 
-next_packet_id(State = #state{last_packet_id = 16#ffff}) ->
+bump_last_packet_id(State = #state{last_packet_id = Id}) ->
-    State#state{last_packet_id = 1};
+    State#state{last_packet_id = next_packet_id(Id)}.
 
-next_packet_id(State = #state{last_packet_id = Id}) ->
+-spec next_packet_id(packet_id()) -> packet_id().
-    State#state{last_packet_id = Id + 1}.
+next_packet_id(?MAX_PACKET_ID) -> 1;
+next_packet_id(Id) -> Id + 1.

src/emqx_connection.erl

@@ -14,20 +14,22 @@
 
 -module(emqx_connection).
 
--behaviour(gen_server).
+-behaviour(gen_statem).
 
 -include("emqx.hrl").
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
 
 -export([start_link/3]).
--export([info/1, attrs/1, stats/1]).
+-export([info/1]).
+-export([attrs/1]).
+-export([stats/1]).
 -export([kick/1]).
 -export([session/1]).
 
-%% gen_server callbacks
+%% gen_statem callbacks
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
+-export([idle/3, connected/3]).
-         code_change/3]).
+-export([init/1, callback_mode/0, code_change/4, terminate/3]).
 
 -record(state, {
           transport,
@@ -37,7 +39,7 @@
           conn_state,
           active_n,
           proto_state,
-          parser_state,
+          parse_state,
           gc_state,
           keepalive,
           enable_stats,
@@ -48,28 +50,29 @@
           idle_timeout
          }).
 
--define(DEFAULT_ACTIVE_N, 100).
+-define(ACTIVE_N, 100).
+-define(HANDLE(T, C, D), handle((T), (C), (D))).
 -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
 
 start_link(Transport, Socket, Options) ->
-    {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Socket, Options]])}.
+    {ok, proc_lib:spawn_link(?MODULE, init, [{Transport, Socket, Options}])}.
 
 %%------------------------------------------------------------------------------
 %% API
 %%------------------------------------------------------------------------------
 
-%% for debug
+%% For debug
 info(CPid) when is_pid(CPid) ->
     call(CPid, info);
 
-info(#state{transport   = Transport,
+info(#state{transport = Transport,
-            socket      = Socket,
+            socket = Socket,
-            peername    = Peername,
+            peername = Peername,
-            sockname    = Sockname,
+            sockname = Sockname,
-            conn_state  = ConnState,
+            conn_state = ConnState,
-            active_n    = ActiveN,
+            active_n = ActiveN,
-            rate_limit  = RateLimit,
+            rate_limit = RateLimit,
-            pub_limit   = PubLimit,
+            pub_limit = PubLimit,
             proto_state = ProtoState}) ->
     ConnInfo = [{socktype, Transport:type(Socket)},
                 {peername, Peername},
@@ -81,10 +84,12 @@ info(#state{transport   = Transport,
     ProtoInfo = emqx_protocol:info(ProtoState),
     lists:usort(lists:append(ConnInfo, ProtoInfo)).
 
-rate_limit_info(undefined) -> #{};
+rate_limit_info(undefined) ->
-rate_limit_info(Limit) -> esockd_rate_limit:info(Limit).
+    #{};
+rate_limit_info(Limit) ->
+    esockd_rate_limit:info(Limit).
 
-%% for dashboard
+%% For dashboard
 attrs(CPid) when is_pid(CPid) ->
     call(CPid, attrs);
 
@@ -100,277 +105,306 @@ attrs(#state{peername = Peername,
 stats(CPid) when is_pid(CPid) ->
     call(CPid, stats);
 
-stats(#state{transport   = Transport,
+stats(#state{transport = Transport,
-             socket      = Socket,
+             socket = Socket,
              proto_state = ProtoState}) ->
-    lists:append([emqx_misc:proc_stats(),
+    SockStats = case Transport:getstat(Socket, ?SOCK_STATS) of
-                  emqx_protocol:stats(ProtoState),
+                    {ok, Ss}   -> Ss;
-                  case Transport:getstat(Socket, ?SOCK_STATS) of
+                    {error, _} -> []
-                      {ok, Ss}   -> Ss;
+                end,
-                      {error, _} -> []
+    lists:append([SockStats,
-                  end]).
+                  emqx_misc:proc_stats(),
+                  emqx_protocol:stats(ProtoState)]).
 
-kick(CPid) -> call(CPid, kick).
+kick(CPid) ->
+    call(CPid, kick).
 
-session(CPid) -> call(CPid, session).
+session(CPid) ->
+    call(CPid, session).
 
 call(CPid, Req) ->
-    gen_server:call(CPid, Req, infinity).
+    gen_statem:call(CPid, Req, infinity).
 
 %%------------------------------------------------------------------------------
-%% gen_server callbacks
+%% gen_statem callbacks
 %%------------------------------------------------------------------------------
 
-init([Transport, RawSocket, Options]) ->
+init({Transport, RawSocket, Options}) ->
-    case Transport:wait(RawSocket) of
+    {ok, Socket} = Transport:wait(RawSocket),
-        {ok, Socket} ->
+    {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
-            Zone = proplists:get_value(zone, Options),
+    {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
-            {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
+    Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
-            {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
+    emqx_logger:set_metadata_peername(esockd_net:format(Peername)),
-            Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
+    Zone = proplists:get_value(zone, Options),
-            RateLimit = init_limiter(proplists:get_value(rate_limit, Options)),
+    RateLimit = init_limiter(proplists:get_value(rate_limit, Options)),
-            PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)),
+    PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)),
-            ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N),
+    ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N),
-            EnableStats = emqx_zone:get_env(Zone, enable_stats, true),
+    EnableStats = emqx_zone:get_env(Zone, enable_stats, true),
-            IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000),
+    IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000),
-            SendFun = send_fun(Transport, Socket),
+    SendFun = fun(Data) -> Transport:async_send(Socket, Data) end,
-            ProtoState = emqx_protocol:init(#{peername => Peername,
+    ProtoState = emqx_protocol:init(#{peername => Peername,
-                                              sockname => Sockname,
+                                      sockname => Sockname,
-                                              peercert => Peercert,
+                                      peercert => Peercert,
-                                              sendfun  => SendFun}, Options),
+                                      sendfun  => SendFun,
-            ParserState = emqx_protocol:parser(ProtoState),
+                                      conn_mod => ?MODULE}, Options),
-            GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
+    ParseState = emqx_protocol:parser(ProtoState),
-            GcState = emqx_gc:init(GcPolicy),
+    GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false),
-            State = run_socket(#state{transport    = Transport,
+    GcState = emqx_gc:init(GcPolicy),
-                                      socket       = Socket,
+    State = #state{transport    = Transport,
-                                      peername     = Peername,
+                   socket       = Socket,
-                                      conn_state   = running,
+                   peername     = Peername,
-                                      active_n     = ActiveN,
+                   conn_state   = running,
-                                      rate_limit   = RateLimit,
+                   active_n     = ActiveN,
-                                      pub_limit    = PubLimit,
+                   rate_limit   = RateLimit,
-                                      proto_state  = ProtoState,
+                   pub_limit    = PubLimit,
-                                      parser_state = ParserState,
+                   proto_state  = ProtoState,
-                                      gc_state     = GcState,
+                   parse_state  = ParseState,
-                                      enable_stats = EnableStats,
+                   gc_state     = GcState,
-                                      idle_timeout = IdleTimout
+                   enable_stats = EnableStats,
-                                     }),
+                   idle_timeout = IdleTimout},
-            ok = emqx_misc:init_proc_mng_policy(Zone),
+    ok = emqx_misc:init_proc_mng_policy(Zone),
-            emqx_logger:set_metadata_peername(esockd_net:format(Peername)),
+    gen_statem:enter_loop(?MODULE, [{hibernate_after, 2 * IdleTimout}],
-            gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}],
+                          idle, State, self(), [IdleTimout]).
-                                  State, self(), IdleTimout);
-        {error, Reason} ->
-            {stop, Reason}
-    end.
 
 init_limiter(undefined) ->
     undefined;
 init_limiter({Rate, Burst}) ->
     esockd_rate_limit:new(Rate, Burst).
 
-send_fun(Transport, Socket) ->
+callback_mode() ->
-    fun(Packet, Options) ->
+    [state_functions, state_enter].
-        Data = emqx_frame:serialize(Packet, Options),
-        try Transport:async_send(Socket, Data) of
-            ok ->
-                emqx_metrics:trans(inc, 'bytes/sent', iolist_size(Data)),
-                ok;
-            Error -> Error
-        catch
-            error:Error ->
-                {error, Error}
-        end
-    end.
 
-handle_call(info, _From, State) ->
+%%------------------------------------------------------------------------------
-    {reply, info(State), State};
+%% Idle state
 
-handle_call(attrs, _From, State) ->
+idle(enter, _, State) ->
-    {reply, attrs(State), State};
+    ok = activate_socket(State),
+    keep_state_and_data;
 
-handle_call(stats, _From, State) ->
+idle(timeout, _Timeout, State) ->
-    {reply, stats(State), State};
+    {stop, idle_timeout, State};
 
-handle_call(kick, _From, State) ->
+idle(cast, {incoming, Packet, PState}, _State) ->
-    {stop, {shutdown, kicked}, ok, State};
+    handle_packet(Packet, fun(NState) ->
+                              {next_state, connected, reset_parser(NState)}
+                          end, PState);
 
-handle_call(session, _From, State = #state{proto_state = ProtoState}) ->
+idle(EventType, Content, State) ->
-    {reply, emqx_protocol:session(ProtoState), State};
+    ?HANDLE(EventType, Content, State).
 
-handle_call(Req, _From, State) ->
+%%------------------------------------------------------------------------------
-    ?LOG(error, "unexpected call: ~p", [Req]),
+%% Connected state
-    {reply, ignored, State}.
 
-handle_cast(Msg, State) ->
+connected(enter, _, _State) ->
-    ?LOG(error, "unexpected cast: ~p", [Msg]),
+    %% What to do?
-    {noreply, State}.
+    keep_state_and_data;
 
-handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) ->
+%% Handle Input
+connected(cast, {incoming, Packet = ?PACKET(Type), PState}, _State) ->
+    _ = emqx_metrics:received(Packet),
+    (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1),
+    handle_packet(Packet, fun(NState) ->
+                              {keep_state, reset_parser(NState)}
+                          end, PState);
+
+%% Handle Output
+connected(info, {deliver, PubOrAck}, State = #state{proto_state = ProtoState}) ->
     case emqx_protocol:deliver(PubOrAck, ProtoState) of
-        {ok, ProtoState1} ->
+        {ok, NProtoState} ->
-            State1 = State#state{proto_state = ProtoState1},
+            NState = State#state{proto_state = NProtoState},
-            {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))};
+            {keep_state, maybe_gc(PubOrAck, NState)};
         {error, Reason} ->
             shutdown(Reason, State)
     end;
 
-handle_info({timeout, Timer, emit_stats},
+%% Start Keepalive
-            State = #state{stats_timer = Timer,
+connected(info, {keepalive, start, Interval},
-                           proto_state = ProtoState,
+          State = #state{transport = Transport, socket = Socket}) ->
-                           gc_state = GcState}) ->
-    emqx_metrics:commit(),
-    emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)),
-    NewState = State#state{stats_timer = undefined},
-    Limits = erlang:get(force_shutdown_policy),
-    case emqx_misc:conn_proc_mng_policy(Limits) of
-        continue ->
-            {noreply, NewState};
-        hibernate ->
-            %% going to hibernate, reset gc stats
-            GcState1 = emqx_gc:reset(GcState),
-            {noreply, NewState#state{gc_state = GcState1}, hibernate};
-        {shutdown, Reason} ->
-            ?LOG(warning, "shutdown due to ~p", [Reason]),
-            shutdown(Reason, NewState)
-    end;
-
-handle_info(timeout, State) ->
-    shutdown(idle_timeout, State);
-
-handle_info({shutdown, Reason}, State) ->
-    shutdown(Reason, State);
-
-handle_info({shutdown, discard, {ClientId, ByPid}}, State) ->
-    ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]),
-    shutdown(discard, State);
-
-handle_info({shutdown, conflict, {ClientId, NewPid}}, State) ->
-    ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]),
-    shutdown(conflict, State);
-
-handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl ->
-    process_incoming(Data, State);
-
-%% Rate limit here, cool:)
-handle_info({tcp_passive, _Sock}, State) ->
-    {noreply, run_socket(ensure_rate_limit(State))};
-%% FIXME Later
-handle_info({ssl_passive, _Sock}, State) ->
-    {noreply, run_socket(ensure_rate_limit(State))};
-
-handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error ->
-    shutdown(Reason, State);
-
-handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed ->
-    shutdown(closed, State);
-
-%% Rate limit timer
-handle_info(activate_sock, State) ->
-    {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})};
-
-handle_info({inet_reply, _Sock, ok}, State) ->
-    {noreply, State};
-
-handle_info({inet_reply, _Sock, {error, Reason}}, State) ->
-    shutdown(Reason, State);
-
-handle_info({keepalive, start, Interval}, State = #state{transport = Transport, socket = Socket}) ->
-    ?LOG(debug, "Keepalive at the interval of ~p", [Interval]),
     StatFun = fun() ->
                 case Transport:getstat(Socket, [recv_oct]) of
                     {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct};
-                    Error                       -> Error
+                    Error -> Error
                 end
-             end,
+              end,
     case emqx_keepalive:start(StatFun, Interval, {keepalive, check}) of
         {ok, KeepAlive} ->
-            {noreply, State#state{keepalive = KeepAlive}};
+            {keep_state, State#state{keepalive = KeepAlive}};
         {error, Error} ->
             shutdown(Error, State)
     end;
 
-handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) ->
+%% Keepalive timer
+connected(info, {keepalive, check}, State = #state{keepalive = KeepAlive}) ->
     case emqx_keepalive:check(KeepAlive) of
         {ok, KeepAlive1} ->
-            {noreply, State#state{keepalive = KeepAlive1}};
+            {keep_state, State#state{keepalive = KeepAlive1}};
         {error, timeout} ->
             shutdown(keepalive_timeout, State);
         {error, Error} ->
             shutdown(Error, State)
     end;
 
-handle_info(Info, State) ->
+connected(EventType, Content, State) ->
-    ?LOG(error, "unexpected info: ~p", [Info]),
+    ?HANDLE(EventType, Content, State).
-    {noreply, State}.
 
-terminate(Reason, #state{transport   = Transport,
+%% Handle call
-                         socket      = Socket,
+handle({call, From}, info, State) ->
-                         keepalive   = KeepAlive,
+    reply(From, info(State), State);
-                         proto_state = ProtoState}) ->
+
+handle({call, From}, attrs, State) ->
+    reply(From, attrs(State), State);
+
+handle({call, From}, stats, State) ->
+    reply(From, stats(State), State);
+
+handle({call, From}, kick, State) ->
+    ok = gen_statem:reply(From, ok),
+    shutdown(kicked, State);
+
+handle({call, From}, session, State = #state{proto_state = ProtoState}) ->
+    reply(From, emqx_protocol:session(ProtoState), State);
+
+handle({call, From}, Req, State) ->
+    ?LOG(error, "unexpected call: ~p", [Req]),
+    reply(From, ignored, State);
+
+%% Handle cast
+handle(cast, Msg, State) ->
+    ?LOG(error, "unexpected cast: ~p", [Msg]),
+    {keep_state, State};
+
+%% Handle Incoming
+handle(info, {Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
+    Oct = iolist_size(Data),
+    ?LOG(debug, "RECV ~p", [Data]),
+    emqx_pd:update_counter(incoming_bytes, Oct),
+    emqx_metrics:trans(inc, 'bytes/received', Oct),
+    NState = ensure_stats_timer(maybe_gc({1, Oct}, State)),
+    process_incoming(Data, [], NState);
+
+handle(info, {Error, _Sock, Reason}, State)
+  when Error == tcp_error; Error == ssl_error ->
+    shutdown(Reason, State);
+
+handle(info, {Closed, _Sock}, State)
+  when Closed == tcp_closed; Closed == ssl_closed ->
+    shutdown(closed, State);
+
+handle(info, {tcp_passive, _Sock}, State) ->
+    %% Rate limit here:)
+    NState = ensure_rate_limit(State),
+    ok = activate_socket(NState),
+    {keep_state, NState};
+
+handle(info, activate_socket, State) ->
+    %% Rate limit timer expired.
+    ok = activate_socket(State),
+    {keep_state, State#state{conn_state = running, limit_timer = undefined}};
+
+handle(info, {inet_reply, _Sock, ok}, State) ->
+    %% something sent
+    {keep_state, ensure_stats_timer(State)};
+
+handle(info, {inet_reply, _Sock, {error, Reason}}, State) ->
+    shutdown(Reason, State);
+
+handle(info, {timeout, Timer, emit_stats},
+       State = #state{stats_timer = Timer,
+                      proto_state = ProtoState,
+                      gc_state = GcState}) ->
+    emqx_metrics:commit(),
+    emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)),
+    NState = State#state{stats_timer = undefined},
+    Limits = erlang:get(force_shutdown_policy),
+    case emqx_misc:conn_proc_mng_policy(Limits) of
+        continue ->
+            {keep_state, NState};
+        hibernate ->
+            %% going to hibernate, reset gc stats
+            GcState1 = emqx_gc:reset(GcState),
+            {keep_state, NState#state{gc_state = GcState1}, hibernate};
+        {shutdown, Reason} ->
+            ?LOG(warning, "shutdown due to ~p", [Reason]),
+            shutdown(Reason, NState)
+    end;
+
+handle(info, {shutdown, discard, {ClientId, ByPid}}, State) ->
+    ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]),
+    shutdown(discard, State);
+
+handle(info, {shutdown, conflict, {ClientId, NewPid}}, State) ->
+    ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]),
+    shutdown(conflict, State);
+
+handle(info, {shutdown, Reason}, State) ->
+    shutdown(Reason, State);
+
+handle(info, Info, State) ->
+    ?LOG(error, "unexpected info: ~p", [Info]),
+    {keep_state, State}.
+
+code_change(_Vsn, State, Data, _Extra) ->
+    {ok, State, Data}.
+
+terminate(Reason, _StateName, #state{transport = Transport,
+                                     socket = Socket,
+                                     keepalive = KeepAlive,
+                                     proto_state = ProtoState}) ->
     ?LOG(debug, "Terminated for ~p", [Reason]),
     Transport:fast_close(Socket),
     emqx_keepalive:cancel(KeepAlive),
     case {ProtoState, Reason} of
         {undefined, _} -> ok;
         {_, {shutdown, Error}} ->
-            emqx_protocol:shutdown(Error, ProtoState);
+            emqx_protocol:terminate(Error, ProtoState);
         {_, Reason} ->
-            emqx_protocol:shutdown(Reason, ProtoState)
+            emqx_protocol:terminate(Reason, ProtoState)
     end.
 
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-%%------------------------------------------------------------------------------
-%% Internals: process incoming, parse and handle packets
 %%------------------------------------------------------------------------------
+%% Process incoming data
 
-process_incoming(Data, State) ->
+process_incoming(<<>>, Packets, State) ->
-    Oct = iolist_size(Data),
+    {keep_state, State, next_events({Packets, State})};
-    ?LOG(debug, "RECV ~p", [Data]),
-    emqx_pd:update_counter(incoming_bytes, Oct),
-    emqx_metrics:trans(inc, 'bytes/received', Oct),
-    case handle_packet(Data, State) of
-        {noreply, State1} ->
-            State2 = maybe_gc({1, Oct}, State1),
-            {noreply, ensure_stats_timer(State2)};
-        Shutdown -> Shutdown
-    end.
 
-%% Parse and handle packets
+process_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
-handle_packet(<<>>, State) ->
+    try emqx_frame:parse(Data, ParseState) of
-    {noreply, State};
+        {ok, Packet, Rest} ->
-
+            process_incoming(Rest, [Packet|Packets], reset_parser(State));
-handle_packet(Data, State = #state{proto_state  = ProtoState,
+        {more, NewParseState} ->
-                                   parser_state = ParserState,
+            {keep_state, State#state{parse_state = NewParseState}, next_events({Packets, State})};
-                                   idle_timeout = IdleTimeout}) ->
-    try emqx_frame:parse(Data, ParserState) of
-        {more, ParserState1} ->
-            {noreply, State#state{parser_state = ParserState1}, IdleTimeout};
-        {ok, Packet = ?PACKET(Type), Rest} ->
-            emqx_metrics:received(Packet),
-            (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1),
-            case emqx_protocol:received(Packet, ProtoState) of
-                {ok, ProtoState1} ->
-                    handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1}));
-                {error, Reason} ->
-                    ?LOG(error, "Process packet error - ~p", [Reason]),
-                    shutdown(Reason, State);
-                {error, Reason, ProtoState1} ->
-                    shutdown(Reason, State#state{proto_state = ProtoState1});
-                {stop, Error, ProtoState1} ->
-                    stop(Error, State#state{proto_state = ProtoState1})
-            end;
         {error, Reason} ->
-            ?LOG(error, "Parse frame error - ~p", [Reason]),
             shutdown(Reason, State)
     catch
-        _:Error ->
+        _:Error:Stk->
-            ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]),
+            ?LOG(error, "Parse failed for ~p~nStacktrace:~p~nError data:~p", [Error, Stk, Data]),
-            shutdown(parse_error, State)
+            shutdown(Error, State)
     end.
 
 reset_parser(State = #state{proto_state = ProtoState}) ->
-    State#state{parser_state = emqx_protocol:parser(ProtoState)}.
+    State#state{parse_state = emqx_protocol:parser(ProtoState)}.
+
+next_events([]) ->
+    [];
+next_events([{Packet, State}]) ->
+    {next_event, cast, {incoming, Packet, State}};
+next_events({Packets, State}) ->
+    [next_events([{Packet, State}]) || Packet <- lists:reverse(Packets)].
+
+%%------------------------------------------------------------------------------
+%% Handle incoming packet
+
+handle_packet(Packet, SuccFun, State = #state{proto_state = ProtoState}) ->
+    case emqx_protocol:received(Packet, ProtoState) of
+        {ok, NProtoState} ->
+            SuccFun(State#state{proto_state = NProtoState});
+        {error, Reason} ->
+            shutdown(Reason, State);
+        {error, Reason, NProtoState} ->
+            shutdown(Reason, State#state{proto_state = NProtoState});
+        {stop, Error, NProtoState} ->
+            stop(Error, State#state{proto_state = NProtoState})
+    end.
 
 %%------------------------------------------------------------------------------
 %% Ensure rate limit
@@ -389,27 +423,27 @@ ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) ->
        {0, Rl1} ->
            ensure_rate_limit(Limiters, setelement(Pos, State, Rl1));
        {Pause, Rl1} ->
-           TRef = erlang:send_after(Pause, self(), activate_sock),
+           TRef = erlang:send_after(Pause, self(), activate_socket),
            setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1)
    end.
 
 %%------------------------------------------------------------------------------
 %% Activate socket
 
-run_socket(State = #state{conn_state = blocked}) ->
+activate_socket(#state{conn_state = blocked}) ->
-    State;
+    ok;
 
-run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) ->
+activate_socket(#state{transport = Transport, socket = Socket, active_n = N}) ->
     TrueOrN = case Transport:is_ssl(Socket) of
                   true  -> true; %% Cannot set '{active, N}' for SSL:(
                   false -> N
               end,
-    ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])),
+    case Transport:setopts(Socket, [{active, TrueOrN}]) of
-    State.
+        ok -> ok;
-
+        {error, Reason} ->
-ensure_ok_or_exit(ok) -> ok;
+            self() ! {shutdown, Reason},
-ensure_ok_or_exit({error, Reason}) ->
+            ok
-    self() ! {shutdown, Reason}.
+    end.
 
 %%------------------------------------------------------------------------------
 %% Ensure stats timer
@@ -418,6 +452,7 @@ ensure_stats_timer(State = #state{enable_stats = true,
                                   stats_timer = undefined,
                                   idle_timeout = IdleTimeout}) ->
     State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)};
+
 ensure_stats_timer(State) -> State.
 
 %%------------------------------------------------------------------------------
@@ -425,20 +460,28 @@ ensure_stats_timer(State) -> State.
 
 maybe_gc(_, State = #state{gc_state = undefined}) ->
     State;
-maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) ->
+maybe_gc({publish, _, #message{payload = Payload}}, State) ->
     Oct = iolist_size(Payload),
     maybe_gc({1, Oct}, State);
+maybe_gc(Packets, State) when is_list(Packets) ->
+    {Cnt, Oct} =
+    lists:unzip([{1, iolist_size(Payload)}
+                 || {publish, _, #message{payload = Payload}} <- Packets]),
+    maybe_gc({lists:sum(Cnt), lists:sum(Oct)}, State);
 maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) ->
     {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt),
     State#state{gc_state = GCSt1};
-maybe_gc(_, State) ->
+maybe_gc(_, State) -> State.
-    State.
 
 %%------------------------------------------------------------------------------
-%% Shutdown or stop
+%% Helper functions
+
+reply(From, Reply, State) ->
+    {keep_state, State, [{reply, From, Reply}]}.
 
 shutdown(Reason, State) ->
     stop({shutdown, Reason}, State).
 
 stop(Reason, State) ->
     {stop, Reason, State}.
+

src/emqx_kernel_sup.erl

@@ -26,7 +26,6 @@ start_link() ->
 init([]) ->
     {ok, {{one_for_one, 10, 100},
           [child_spec(emqx_pool_sup, supervisor),
-           child_spec(emqx_alarm_mgr, worker),
            child_spec(emqx_hooks, worker),
            child_spec(emqx_stats, worker),
            child_spec(emqx_metrics, worker),

src/emqx_listeners.erl

@@ -56,7 +56,7 @@ start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws ->
 
 %% Start MQTT/WSS listener
 start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss ->
-    Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options),  emqx_ws_connection, Options}]}]),
+    Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options), emqx_ws_connection, Options}]}]),
     start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, ranch_opts(Options), Dispatch).
 
 start_mqtt_listener(Name, ListenOn, Options) ->

src/emqx_local_bridge.erl

@@ -1,157 +0,0 @@
-%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-
--module(emqx_local_bridge).
-
--behaviour(gen_server).
-
--include("emqx.hrl").
--include("emqx_mqtt.hrl").
-
--export([start_link/5]).
-
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
-         code_change/3]).
-
--define(PING_DOWN_INTERVAL, 1000).
-
--record(state, {pool, id,
-                node, subtopic,
-                qos                = ?QOS_0,
-                topic_suffix       = <<>>,
-                topic_prefix       = <<>>,
-                mqueue            :: emqx_mqueue:mqueue(),
-                max_queue_len      = 10000,
-                ping_down_interval = ?PING_DOWN_INTERVAL,
-                status             = up}).
-
--type(option() :: {qos, emqx_mqtt_types:qos()} |
-                  {topic_suffix, binary()} |
-                  {topic_prefix, binary()} |
-                  {max_queue_len, pos_integer()} |
-                  {ping_down_interval, pos_integer()}).
-
--export_type([option/0]).
-
-%% @doc Start a bridge
--spec(start_link(term(), pos_integer(), atom(), binary(), [option()])
-      -> {ok, pid()} | ignore | {error, term()}).
-start_link(Pool, Id, Node, Topic, Options) ->
-    gen_server:start_link(?MODULE, [Pool, Id, Node, Topic, Options], [{hibernate_after, 5000}]).
-
-%%------------------------------------------------------------------------------
-%% gen_server callbacks
-%%------------------------------------------------------------------------------
-
-init([Pool, Id, Node, Topic, Options]) ->
-    process_flag(trap_exit, true),
-    true = gproc_pool:connect_worker(Pool, {Pool, Id}),
-    case net_kernel:connect_node(Node) of
-        true ->
-            true = erlang:monitor_node(Node, true),
-            Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]),
-            emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}),
-            State = parse_opts(Options, #state{node = Node, subtopic = Topic}),
-            MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len,
-                                        store_qos0 => true}),
-            {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}};
-        false ->
-            {stop, {cannot_connect_node, Node}}
-    end.
-
-parse_opts([], State) ->
-    State;
-parse_opts([{qos, QoS} | Opts], State) ->
-    parse_opts(Opts, State#state{qos = QoS});
-parse_opts([{topic_suffix, Suffix} | Opts], State) ->
-    parse_opts(Opts, State#state{topic_suffix= Suffix});
-parse_opts([{topic_prefix, Prefix} | Opts], State) ->
-    parse_opts(Opts, State#state{topic_prefix = Prefix});
-parse_opts([{max_queue_len, Len} | Opts], State) ->
-    parse_opts(Opts, State#state{max_queue_len = Len});
-parse_opts([{ping_down_interval, Interval} | Opts], State) ->
-    parse_opts(Opts, State#state{ping_down_interval = Interval});
-parse_opts([_Opt | Opts], State) ->
-    parse_opts(Opts, State).
-
-handle_call(Req, _From, State) ->
-    emqx_logger:error("[Bridge] unexpected call: ~p", [Req]),
-    {reply, ignored, State}.
-
-handle_cast(Msg, State) ->
-    emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]),
-    {noreply, State}.
-
-handle_info({dispatch, _Topic, Msg}, State = #state{mqueue = Q, status = down}) ->
-    %% TODO: how to drop???
-    {_Dropped, NewQ} = emqx_mqueue:in(Msg, Q),
-    {noreply, State#state{mqueue = NewQ}};
-
-handle_info({dispatch, _Topic, Msg}, State = #state{node = Node, status = up}) ->
-    emqx_rpc:cast(Node, emqx_broker, publish, [transform(Msg, State)]),
-    {noreply, State};
-
-handle_info({nodedown, Node}, State = #state{node = Node, ping_down_interval = Interval}) ->
-    emqx_logger:warning("[Bridge] node down: ~s", [Node]),
-    erlang:send_after(Interval, self(), ping_down_node),
-    {noreply, State#state{status = down}, hibernate};
-
-handle_info({nodeup, Node}, State = #state{node = Node}) ->
-    %% TODO: Really fast??
-    case emqx:is_running(Node) of
-        true -> emqx_logger:warning("[Bridge] Node up: ~s", [Node]),
-                {noreply, dequeue(State#state{status = up})};
-        false -> self() ! {nodedown, Node},
-                 {noreply, State#state{status = down}}
-    end;
-
-handle_info(ping_down_node, State = #state{node = Node, ping_down_interval = Interval}) ->
-    Self = self(),
-    spawn_link(fun() ->
-                 case net_kernel:connect_node(Node) of
-                     true -> Self ! {nodeup, Node};
-                     false -> erlang:send_after(Interval, Self, ping_down_node)
-                 end
-               end),
-    {noreply, State};
-
-handle_info({'EXIT', _Pid, normal}, State) ->
-    {noreply, State};
-
-handle_info(Info, State) ->
-    emqx_logger:error("[Bridge] unexpected info: ~p", [Info]),
-    {noreply, State}.
-
-terminate(_Reason, #state{pool = Pool, id = Id}) ->
-    gproc_pool:disconnect_worker(Pool, {Pool, Id}).
-
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-%%--------------------------------------------------------------------
-%% Internal functions
-%%--------------------------------------------------------------------
-
-dequeue(State = #state{mqueue = MQ}) ->
-    case emqx_mqueue:out(MQ) of
-        {empty, MQ1} ->
-            State#state{mqueue = MQ1};
-        {{value, Msg}, MQ1} ->
-            handle_info({dispatch, Msg#message.topic, Msg}, State),
-            dequeue(State#state{mqueue = MQ1})
-    end.
-
-transform(Msg = #message{topic = Topic}, #state{topic_prefix = Prefix, topic_suffix = Suffix}) ->
-    Msg#message{topic = <<Prefix/binary, Topic/binary, Suffix/binary>>}.
-

src/emqx_local_bridge_sup_sup.erl

@@ -1,74 +0,0 @@
-%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-
--module(emqx_local_bridge_sup_sup).
-
--behavior(supervisor).
-
--include("emqx.hrl").
-
--export([start_link/0, bridges/0]).
--export([start_bridge/2, start_bridge/3, stop_bridge/2]).
-
-%% Supervisor callbacks
--export([init/1]).
-
--define(CHILD_ID(Node, Topic), {bridge_sup, Node, Topic}).
-
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
-%% @doc List all bridges
--spec(bridges() -> [{node(), emqx_topic:topic(), pid()}]).
-bridges() ->
-    [{Node, Topic, Pid} || {?CHILD_ID(Node, Topic), Pid, supervisor, _}
-                           <- supervisor:which_children(?MODULE)].
-
-%% @doc Start a bridge
--spec(start_bridge(node(), emqx_topic:topic()) -> {ok, pid()} | {error, term()}).
-start_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) ->
-    start_bridge(Node, Topic, []).
-
--spec(start_bridge(node(), emqx_topic:topic(), [emqx_bridge:option()])
-      -> {ok, pid()} | {error, term()}).
-start_bridge(Node, _Topic, _Options) when Node =:= node() ->
-    {error, bridge_to_self};
-start_bridge(Node, Topic, Options) when is_atom(Node), is_binary(Topic) ->
-    Options1 = emqx_misc:merge_opts(emqx_config:get_env(bridge, []), Options),
-    supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)).
-
-%% @doc Stop a bridge
--spec(stop_bridge(node(), emqx_topic:topic()) -> ok | {error, term()}).
-stop_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) ->
-    ChildId = ?CHILD_ID(Node, Topic),
-    case supervisor:terminate_child(?MODULE, ChildId) of
-        ok    -> supervisor:delete_child(?MODULE, ChildId);
-        Error -> Error
-    end.
-
-%%------------------------------------------------------------------------------
-%% Supervisor callbacks
-%%------------------------------------------------------------------------------
-
-init([]) ->
-    {ok, {{one_for_one, 10, 3600}, []}}.
-
-bridge_spec(Node, Topic, Options) ->
-    #{id       => ?CHILD_ID(Node, Topic),
-      start    => {emqx_local_bridge_sup, start_link, [Node, Topic, Options]},
-      restart  => permanent,
-      shutdown => infinity,
-      type     => supervisor,
-      modules  => [emqx_local_bridge_sup]}.
-

src/emqx_logger_formatter.erl

@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2017-2013-2019. All Rights Reserved.
+%% Copyright Ericsson AB 2013-2019. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

src/emqx_logger_handler.erl

@@ -0,0 +1,42 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_logger_handler).
+
+-export([log/2]).
+-export([init/0]).
+
+init() ->
+    logger:add_handler(emqx_logger_handler, 
+                       emqx_logger_handler, 
+                       #{level => error,
+                         filters => [{easy_filter, {fun filter_by_level/2, []}}],
+                         filters_default => stop}).
+
+-spec log(LogEvent, Config) -> ok when LogEvent :: logger:log_event(), Config :: logger:handler_config().
+log(#{msg := {report, #{report := [{supervisor, SupName},
+                                   {errorContext, Error},
+                                   {reason, Reason},
+                                   {offender, _}]}}}, _Config) ->
+    alarm_handler:set_alarm({supervisor_report, [{supervisor, SupName},
+                                                 {errorContext, Error},
+                                                 {reason, Reason}]}),
+    ok;
+log(_LogEvent, _Config) ->
+    ok.
+
+filter_by_level(LogEvent = #{level := error}, _Extra) ->
+    LogEvent;
+filter_by_level(_LogEvent, _Extra) ->
+    stop.

src/emqx_mqueue.erl

@@ -67,7 +67,7 @@
                      default_priority => highest | lowest,
                      store_qos0 => boolean()
                     }).
--type(message() :: pemqx_types:message()).
+-type(message() :: emqx_types:message()).
 
 -type(stat() :: {len, non_neg_integer()}
               | {max_len, non_neg_integer()}

src/emqx_os_mon.erl

@@ -0,0 +1,153 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_os_mon).
+
+-behaviour(gen_server).
+
+-include("logger.hrl").
+
+-export([start_link/1]).
+
+-export([init/1,
+         handle_call/3,
+         handle_cast/2,
+         handle_info/2,
+         terminate/2,
+         code_change/3]).
+
+-export([get_cpu_check_interval/0,
+         set_cpu_check_interval/1,
+         get_cpu_high_watermark/0,
+         set_cpu_high_watermark/1,
+         get_cpu_low_watermark/0,
+         set_cpu_low_watermark/1,
+         get_mem_check_interval/0,
+         set_mem_check_interval/1,
+         get_sysmem_high_watermark/0,
+         set_sysmem_high_watermark/1,
+         get_procmem_high_watermark/0,
+         set_procmem_high_watermark/1]).
+
+-define(OS_MON, ?MODULE).
+
+%%----------------------------------------------------------------------
+%% API
+%%----------------------------------------------------------------------
+
+start_link(Opts) ->
+    gen_server:start_link({local, ?OS_MON}, ?MODULE, [Opts], []).
+
+get_cpu_check_interval() ->
+    call(get_cpu_check_interval).
+
+set_cpu_check_interval(Seconds) ->
+    call({set_cpu_check_interval, Seconds}).
+
+get_cpu_high_watermark() ->
+    call(get_cpu_high_watermark).
+
+set_cpu_high_watermark(Float) ->
+    call({set_cpu_high_watermark, Float}).
+
+get_cpu_low_watermark() ->
+    call(get_cpu_low_watermark).
+
+set_cpu_low_watermark(Float) ->
+    call({set_cpu_low_watermark, Float}).
+
+get_mem_check_interval() ->
+    memsup:get_check_interval() div 1000.
+
+set_mem_check_interval(Seconds) ->
+    memsup:set_check_interval(Seconds div 60).
+
+get_sysmem_high_watermark() ->
+    memsup:get_sysmem_high_watermark() / 100.
+
+set_sysmem_high_watermark(Float) ->
+    memsup:set_sysmem_high_watermark(Float).
+
+get_procmem_high_watermark() ->
+    memsup:get_procmem_high_watermark() / 100.
+
+set_procmem_high_watermark(Float) ->
+    memsup:set_procmem_high_watermark(Float).
+
+%%----------------------------------------------------------------------
+%% gen_server callbacks
+%%----------------------------------------------------------------------
+
+init([Opts]) ->
+    _ = cpu_sup:util(),
+    set_mem_check_interval(proplists:get_value(mem_check_interval, Opts, 60)),
+    set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts, 0.70)),
+    set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts, 0.05)),
+    {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts, 0.80),
+                              cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts, 0.60),
+                              cpu_check_interval => proplists:get_value(cpu_check_interval, Opts, 60),
+                              timer => undefined})}.
+
+handle_call(get_cpu_check_interval, _From, State) ->
+    {reply, maps:get(cpu_check_interval, State, undefined), State};
+handle_call({set_cpu_check_interval, Seconds}, _From, State) ->
+    {reply, ok, State#{cpu_check_interval := Seconds}};
+
+handle_call(get_cpu_high_watermark, _From, State) ->
+    {reply, maps:get(cpu_high_watermark, State, undefined), State};
+handle_call({set_cpu_high_watermark, Float}, _From, State) ->
+    {reply, ok, State#{cpu_high_watermark := Float}};
+
+handle_call(get_cpu_low_watermark, _From, State) ->
+    {reply, maps:get(cpu_low_watermark, State, undefined), State};
+handle_call({set_cpu_low_watermark, Float}, _From, State) ->
+    {reply, ok, State#{cpu_low_watermark := Float}};
+
+handle_call(_Request, _From, State) ->
+    {reply, ok, State}.
+
+handle_cast(_Request, State) ->
+    {noreply, State}.
+
+handle_info({timeout, Timer, check}, State = #{timer := Timer, 
+                                               cpu_high_watermark := CPUHighWatermark,
+                                               cpu_low_watermark := CPULowWatermark}) ->
+    case cpu_sup:util() of
+        0 ->
+            {noreply, State#{timer := undefined}};
+        {error, Reason} ->
+            ?LOG(warning, "Failed to get cpu utilization: ~p", [Reason]),
+            {noreply, ensure_check_timer(State)};
+        Busy when Busy / 100 >= CPUHighWatermark ->
+            alarm_handler:set_alarm({cpu_high_watermark, Busy}),
+            {noreply, ensure_check_timer(State)};
+        Busy when Busy / 100 < CPULowWatermark ->
+            alarm_handler:clear_alarm(cpu_high_watermark),
+            {noreply, ensure_check_timer(State)}
+    end.
+
+terminate(_Reason, #{timer := Timer}) ->
+    emqx_misc:cancel_timer(Timer).
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%----------------------------------------------------------------------
+%% Internal functions
+%%----------------------------------------------------------------------
+call(Req) ->
+    gen_server:call(?OS_MON, Req, infinity).
+
+ensure_check_timer(State = #{cpu_check_interval := Interval}) ->
+    State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}.

src/emqx_protocol.erl

@@ -29,10 +29,10 @@
 -export([parser/1]).
 -export([session/1]).
 -export([received/2]).
--export([process_packet/2]).
+-export([process/2]).
 -export([deliver/2]).
 -export([send/2]).
--export([shutdown/2]).
+-export([terminate/2]).
 
 -export_type([state/0]).
 
@@ -53,6 +53,8 @@
           clean_start,
           topic_aliases,
           packet_size,
+          will_topic,
+          will_msg,
           keepalive,
           mountpoint,
           is_super,
@@ -65,7 +67,8 @@
           connected,
           connected_at,
           ignore_loop,
-          topic_alias_maximum
+          topic_alias_maximum,
+          conn_mod
         }).
 
 -opaque(state() :: #pstate{}).
@@ -82,7 +85,7 @@
 %%------------------------------------------------------------------------------
 
 -spec(init(map(), list()) -> state()).
-init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) ->
+init(SocketOpts = #{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options)  ->
     Zone = proplists:get_value(zone, Options),
     #pstate{zone                = Zone,
             sendfun             = SendFun,
@@ -107,7 +110,8 @@ init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options)
             send_stats          = #{msg => 0, pkt => 0},
             connected           = false,
             ignore_loop         = emqx_config:get_env(mqtt_ignore_loop_deliver, false),
-            topic_alias_maximum = #{to_client => 0, from_client => 0}}.
+            topic_alias_maximum = #{to_client => 0, from_client => 0},
+            conn_mod            = maps:get(conn_mod, SocketOpts, undefined)}.
 
 init_username(Peercert, Options) ->
     case proplists:get_value(peer_cert_as_username, Options) of
@@ -130,11 +134,13 @@ info(PState = #pstate{conn_props    = ConnProps,
                       ack_props     = AckProps,
                       session       = Session,
                       topic_aliases = Aliases,
+                      will_msg      = WillMsg,
                       enable_acl    = EnableAcl}) ->
     attrs(PState) ++ [{conn_props, ConnProps},
                       {ack_props, AckProps},
                       {session, Session},
                       {topic_aliases, Aliases},
+                      {will_msg, WillMsg},
                       {enable_acl, EnableAcl}].
 
 attrs(#pstate{zone         = Zone,
@@ -149,7 +155,8 @@ attrs(#pstate{zone         = Zone,
               mountpoint   = Mountpoint,
               is_super     = IsSuper,
               is_bridge    = IsBridge,
-              connected_at = ConnectedAt}) ->
+              connected_at = ConnectedAt,
+              conn_mod     = ConnMod}) ->
     [{zone, Zone},
      {client_id, ClientId},
      {username, Username},
@@ -162,7 +169,8 @@ attrs(#pstate{zone         = Zone,
      {mountpoint, Mountpoint},
      {is_super, IsSuper},
      {is_bridge, IsBridge},
-     {connected_at, ConnectedAt}].
+     {connected_at, ConnectedAt},
+     {conn_mod, ConnMod}].
 
 attr(max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}) ->
     get_property('Receive-Maximum', ConnProps, 65535);
@@ -218,15 +226,16 @@ parser(#pstate{packet_size = Size, proto_ver = Ver}) ->
 %% Packet Received
 %%------------------------------------------------------------------------------
 
-set_protover(?CONNECT_PACKET(#mqtt_packet_connect{
+set_protover(?CONNECT_PACKET(#mqtt_packet_connect{proto_ver = ProtoVer}), PState) ->
-                                proto_ver = ProtoVer}),
+    PState#pstate{proto_ver = ProtoVer};
-             PState) ->
-    PState#pstate{ proto_ver = ProtoVer };
 set_protover(_Packet, PState) ->
     PState.
 
--spec(received(emqx_mqtt_types:packet(), state()) ->
+-spec(received(emqx_mqtt_types:packet(), state())
-    {ok, state()} | {error, term()} | {error, term(), state()} | {stop, term(), state()}).
+      -> {ok, state()}
+       | {error, term()}
+       | {error, term(), state()}
+       | {stop, term(), state()}).
 received(?PACKET(Type), PState = #pstate{connected = false}) when Type =/= ?CONNECT ->
     {error, proto_not_connected, PState};
 
@@ -234,15 +243,15 @@ received(?PACKET(?CONNECT), PState = #pstate{connected = true}) ->
     {error, proto_unexpected_connect, PState};
 
 received(Packet = ?PACKET(Type), PState) ->
-    PState1 = set_protover(Packet, PState),
     trace(recv, Packet),
+    PState1 = set_protover(Packet, PState),
     try emqx_packet:validate(Packet) of
         true ->
             case preprocess_properties(Packet, PState1) of
+                {ok, Packet1, PState2} ->
+                    process(Packet1, inc_stats(recv, Type, PState2));
                 {error, ReasonCode} ->
-                    {error, ReasonCode, PState1};
+                    {error, ReasonCode, PState1}
-                {Packet1, PState2} ->
-                    process_packet(Packet1, inc_stats(recv, Type, PState2))
             end
     catch
         error:protocol_error ->
@@ -268,13 +277,14 @@ received(Packet = ?PACKET(Type), PState) ->
 %%------------------------------------------------------------------------------
 %% Preprocess MQTT Properties
 %%------------------------------------------------------------------------------
+
 preprocess_properties(Packet = #mqtt_packet{
                                    variable = #mqtt_packet_connect{
                                                   properties = #{'Topic-Alias-Maximum' := ToClient}
                                               }
                                },
                       PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) ->
-    {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}};
+    {ok, Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}};
 
 %% Subscription Identifier
 preprocess_properties(Packet = #mqtt_packet{
@@ -285,7 +295,7 @@ preprocess_properties(Packet = #mqtt_packet{
                                  },
                       PState = #pstate{proto_ver = ?MQTT_PROTO_V5}) ->
     TopicFilters1 = [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters],
-    {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState};
+    {ok, Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState};
 
 %% Topic Alias Mapping
 preprocess_properties(#mqtt_packet{
@@ -306,8 +316,8 @@ preprocess_properties(Packet = #mqtt_packet{
                                        topic_alias_maximum = #{from_client := TopicAliasMaximum}}) ->
     case AliasId =< TopicAliasMaximum of
         true ->
-            {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{
+            {ok, Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{
-                                               topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState};
+                                                 topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState};
         false ->
             deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState),
             {error, ?RC_TOPIC_ALIAS_INVALID}
@@ -323,28 +333,28 @@ preprocess_properties(Packet = #mqtt_packet{
                                        topic_alias_maximum = #{from_client := TopicAliasMaximum}}) ->
     case AliasId =< TopicAliasMaximum of
         true ->
-            {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}};
+            {ok, Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}};
         false ->
             deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState),
             {error, ?RC_TOPIC_ALIAS_INVALID}
     end;
 
 preprocess_properties(Packet, PState) ->
-    {Packet, PState}.
+    {ok, Packet, PState}.
 
 %%------------------------------------------------------------------------------
 %% Process MQTT Packet
 %%------------------------------------------------------------------------------
-process_packet(?CONNECT_PACKET(
+process(?CONNECT_PACKET(
-                  #mqtt_packet_connect{proto_name  = ProtoName,
+           #mqtt_packet_connect{proto_name  = ProtoName,
-                                       proto_ver   = ProtoVer,
+                                proto_ver   = ProtoVer,
-                                       is_bridge   = IsBridge,
+                                is_bridge   = IsBridge,
-                                       clean_start = CleanStart,
+                                clean_start = CleanStart,
-                                       keepalive   = Keepalive,
+                                keepalive   = Keepalive,
-                                       properties  = ConnProps,
+                                properties  = ConnProps,
-                                       client_id   = ClientId,
+                                client_id   = ClientId,
-                                       username    = Username,
+                                username    = Username,
-                                       password    = Password} = ConnPkt), PState) ->
+                                password    = Password} = ConnPkt), PState) ->
 
     NewClientId = maybe_use_username_as_clientid(ClientId, Username, PState),
 
@@ -394,17 +404,17 @@ process_packet(?CONNECT_PACKET(
               {ReasonCode, PState1}
       end);
 
-process_packet(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PState) ->
+process(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PState) ->
     case check_publish(Packet, PState) of
         {ok, PState1} ->
             do_publish(Packet, PState1);
         {error, ReasonCode} ->
             ?LOG(warning, "Cannot publish qos0 message to ~s for ~s",
-                [Topic, emqx_reason_codes:text(ReasonCode)]),
+                 [Topic, emqx_reason_codes:text(ReasonCode)]),
             do_acl_deny_action(Packet, ReasonCode, PState)
     end;
 
-process_packet(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PState) ->
+process(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PState) ->
     case check_publish(Packet, PState) of
         {ok, PState1} ->
             do_publish(Packet, PState1);
@@ -414,30 +424,28 @@ process_packet(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PSta
             case deliver({puback, PacketId, ReasonCode}, PState) of
                 {ok, PState1} ->
                     do_acl_deny_action(Packet, ReasonCode, PState1);
-                Error ->
+                Error -> Error
-                    Error
             end
     end;
 
-process_packet(Packet = ?PUBLISH_PACKET(?QOS_2, Topic, PacketId, _Payload), PState) ->
+process(Packet = ?PUBLISH_PACKET(?QOS_2, Topic, PacketId, _Payload), PState) ->
     case check_publish(Packet, PState) of
         {ok, PState1} ->
             do_publish(Packet, PState1);
         {error, ReasonCode} ->
             ?LOG(warning, "Cannot publish qos2 message to ~s for ~s",
-                [Topic, emqx_reason_codes:text(ReasonCode)]),
+                 [Topic, emqx_reason_codes:text(ReasonCode)]),
             case deliver({pubrec, PacketId, ReasonCode}, PState) of
                 {ok, PState1} ->
                     do_acl_deny_action(Packet, ReasonCode, PState1);
-                Error ->
+                Error -> Error
-                    Error
             end
     end;
 
-process_packet(?PUBACK_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
+process(?PUBACK_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
     {ok = emqx_session:puback(SPid, PacketId, ReasonCode), PState};
 
-process_packet(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
+process(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
     case emqx_session:pubrec(SPid, PacketId, ReasonCode) of
         ok ->
             send(?PUBREL_PACKET(PacketId), PState);
@@ -445,7 +453,7 @@ process_packet(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session =
             send(?PUBREL_PACKET(PacketId, NotFound), PState)
     end;
 
-process_packet(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
+process(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
     case emqx_session:pubrel(SPid, PacketId, ReasonCode) of
         ok ->
             send(?PUBCOMP_PACKET(PacketId), PState);
@@ -453,22 +461,22 @@ process_packet(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session =
             send(?PUBCOMP_PACKET(PacketId, NotFound), PState)
     end;
 
-process_packet(?PUBCOMP_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
+process(?PUBCOMP_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) ->
     {ok = emqx_session:pubcomp(SPid, PacketId, ReasonCode), PState};
 
-process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters),
+process(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters),
-               PState = #pstate{session = SPid, mountpoint = Mountpoint,
+        PState = #pstate{session = SPid, mountpoint = Mountpoint,
-                                proto_ver = ProtoVer, is_bridge = IsBridge,
+                         proto_ver = ProtoVer, is_bridge = IsBridge,
-                                ignore_loop = IgnoreLoop}) ->
+                         ignore_loop = IgnoreLoop}) ->
-    RawTopicFilters1 =  if ProtoVer < ?MQTT_PROTO_V5 ->
+    RawTopicFilters1 = if ProtoVer < ?MQTT_PROTO_V5 ->
-                            IfIgnoreLoop = case IgnoreLoop of true -> 1; false -> 0 end,
+                           IfIgnoreLoop = case IgnoreLoop of true -> 1; false -> 0 end,
-                            case IsBridge of
+                           case IsBridge of
-                                true -> [{RawTopic, SubOpts#{rap => 1, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters];
+                               true -> [{RawTopic, SubOpts#{rap => 1, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters];
-                                false -> [{RawTopic, SubOpts#{rap => 0, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters]
+                               false -> [{RawTopic, SubOpts#{rap => 0, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters]
-                            end;
+                           end;
-                           true ->
+                          true ->
-                               RawTopicFilters
+                              RawTopicFilters
-                        end,
+                       end,
     case check_subscribe(
            parse_topic_filters(?SUBSCRIBE, RawTopicFilters1), PState) of
         {ok, TopicFilters} ->
@@ -483,15 +491,14 @@ process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters)
                     deliver({suback, PacketId, ReasonCodes}, PState)
             end;
         {error, TopicFilters} ->
-            {ReverseSubTopics, ReverseReasonCodes} =
+            {SubTopics, ReasonCodes} =
-                lists:foldl(fun({Topic, #{rc := ?RC_SUCCESS}}, {Topics, Codes}) ->
+                lists:foldr(fun({Topic, #{rc := ?RC_SUCCESS}}, {Topics, Codes}) ->
                                     {[Topic|Topics], [?RC_IMPLEMENTATION_SPECIFIC_ERROR | Codes]};
                                 ({Topic, #{rc := Code}}, {Topics, Codes}) ->
                                     {[Topic|Topics], [Code|Codes]}
                             end, {[], []}, TopicFilters),
-            {SubTopics, ReasonCodes} = {lists:reverse(ReverseSubTopics), lists:reverse(ReverseReasonCodes)},
             ?LOG(warning, "Cannot subscribe ~p for ~p",
-                [SubTopics, [emqx_reason_codes:text(R) || R <- ReasonCodes]]),
+                 [SubTopics, [emqx_reason_codes:text(R) || R <- ReasonCodes]]),
             case deliver({suback, PacketId, ReasonCodes}, PState) of
                 {ok, PState1} ->
                     do_acl_deny_action(Packet, ReasonCodes, PState1);
@@ -500,8 +507,8 @@ process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters)
             end
     end;
 
-process_packet(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters),
+process(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters),
-               PState = #pstate{session = SPid, mountpoint = MountPoint}) ->
+        PState = #pstate{session = SPid, mountpoint = MountPoint}) ->
     case emqx_hooks:run('client.unsubscribe', [credentials(PState)],
                         parse_topic_filters(?UNSUBSCRIBE, RawTopicFilters)) of
         {ok, TopicFilters} ->
@@ -514,22 +521,25 @@ process_packet(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters),
             deliver({unsuback, PacketId, ReasonCodes}, PState)
     end;
 
-process_packet(?PACKET(?PINGREQ), PState) ->
+process(?PACKET(?PINGREQ), PState) ->
     send(?PACKET(?PINGRESP), PState);
 
-process_packet(?DISCONNECT_PACKET(?RC_SUCCESS, #{'Session-Expiry-Interval' := Interval}),
+process(?DISCONNECT_PACKET(?RC_SUCCESS, #{'Session-Expiry-Interval' := Interval}),
-                PState = #pstate{session = SPid, conn_props = #{'Session-Expiry-Interval' := OldInterval}}) ->
+        PState = #pstate{session = SPid, conn_props = #{'Session-Expiry-Interval' := OldInterval}}) ->
     case Interval =/= 0 andalso OldInterval =:= 0 of
         true ->
             deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState),
-            {error, protocol_error, PState};
+            {error, protocol_error, PState#pstate{will_msg = undefined}};
         false ->
             emqx_session:update_expiry_interval(SPid, Interval),
-            {stop, normal, PState}
+            %% Clean willmsg
+            {stop, normal, PState#pstate{will_msg = undefined}}
     end;
-process_packet(?DISCONNECT_PACKET(?RC_SUCCESS), PState) ->
+
-    {stop, normal, PState};
+process(?DISCONNECT_PACKET(?RC_SUCCESS), PState) ->
-process_packet(?DISCONNECT_PACKET(_), PState) ->
+    {stop, normal, PState#pstate{will_msg = undefined}};
+
+process(?DISCONNECT_PACKET(_), PState) ->
     {stop, {shutdown, abnormal_disconnet}, PState}.
 
 %%------------------------------------------------------------------------------
@@ -562,15 +572,16 @@ do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId),
 
 puback(?QOS_0, _PacketId, _Result, PState) ->
     {ok, PState};
-puback(?QOS_1, PacketId, [], PState) ->
+puback(?QOS_1, PacketId, {ok, []}, PState) ->
     deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState);
-puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch?
+%%TODO: calc the deliver count?
+puback(?QOS_1, PacketId, {ok, _Result}, PState) ->
     deliver({puback, PacketId, ?RC_SUCCESS}, PState);
 puback(?QOS_1, PacketId, {error, ReasonCode}, PState) ->
     deliver({puback, PacketId, ReasonCode}, PState);
-puback(?QOS_2, PacketId, [], PState) ->
+puback(?QOS_2, PacketId, {ok, []}, PState) ->
     deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState);
-puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch?
+puback(?QOS_2, PacketId, {ok, _Result}, PState) ->
     deliver({pubrec, PacketId, ?RC_SUCCESS}, PState);
 puback(?QOS_2, PacketId, {error, ReasonCode}, PState) ->
     deliver({pubrec, PacketId, ReasonCode}, PState).
@@ -579,7 +590,17 @@ puback(?QOS_2, PacketId, {error, ReasonCode}, PState) ->
 %% Deliver Packet -> Client
 %%------------------------------------------------------------------------------
 
--spec(deliver(tuple(), state()) -> {ok, state()} | {error, term()}).
+-spec(deliver(list(tuple()) | tuple(), state()) -> {ok, state()} | {error, term()}).
+deliver([], PState) ->
+    {ok, PState};
+deliver([Pub|More], PState) ->
+    case deliver(Pub, PState) of
+        {ok, PState1} ->
+            deliver(More, PState1);
+        {error, _} = Error ->
+            Error
+    end;
+
 deliver({connack, ReasonCode}, PState) ->
     send(?CONNACK_PACKET(ReasonCode), PState);
 
@@ -666,11 +687,13 @@ deliver({disconnect, _ReasonCode}, PState) ->
 %% Send Packet to Client
 
 -spec(send(emqx_mqtt_types:packet(), state()) -> {ok, state()} | {error, term()}).
-send(Packet = ?PACKET(Type), PState = #pstate{proto_ver = Ver, sendfun = SendFun}) ->
+send(Packet = ?PACKET(Type), PState = #pstate{proto_ver = Ver, sendfun = Send}) ->
-    trace(send, Packet),
+    Data = emqx_frame:serialize(Packet, #{version => Ver}),
-    case SendFun(Packet, #{version => Ver}) of
+    case Send(Data) of
         ok ->
+            trace(send, Packet),
             emqx_metrics:sent(Packet),
+            emqx_metrics:trans(inc, 'bytes/sent', iolist_size(Data)),
             {ok, inc_stats(send, Type, PState)};
         {error, Reason} ->
             {error, Reason}
@@ -809,14 +832,13 @@ check_will_topic(#mqtt_packet_connect{will_topic = WillTopic} = ConnPkt, PState)
             {error, ?RC_TOPIC_NAME_INVALID}
     end.
 
-check_will_acl(_ConnPkt, #pstate{enable_acl = EnableAcl})
+check_will_acl(_ConnPkt, #pstate{enable_acl = EnableAcl}) when not EnableAcl ->
-  when not EnableAcl ->
     ok;
 check_will_acl(#mqtt_packet_connect{will_topic = WillTopic}, PState) ->
     case emqx_access_control:check_acl(credentials(PState), publish, WillTopic) of
         allow -> ok;
         deny ->
-            ?LOG(warning, "Will message (to ~s) validation failed, acl denied", [WillTopic]),
+            ?LOG(warning, "Cannot publish will message to ~p for acl denied", [WillTopic]),
             {error, ?RC_NOT_AUTHORIZED}
     end.
 
@@ -825,7 +847,7 @@ check_publish(Packet, PState) ->
                      fun check_pub_acl/2], Packet, PState).
 
 check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain},
-                            variable = #mqtt_packet_publish{ properties = _Properties}},
+                            variable = #mqtt_packet_publish{properties = _Properties}},
                #pstate{zone = Zone}) ->
     emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}).
 
@@ -892,15 +914,15 @@ inc_stats(Type, Stats = #{pkt := PktCnt, msg := MsgCnt}) ->
                                          false -> MsgCnt
                                      end}.
 
-shutdown(_Reason, #pstate{client_id = undefined}) ->
+terminate(_Reason, #pstate{client_id = undefined}) ->
     ok;
-shutdown(_Reason, #pstate{connected = false}) ->
+terminate(_Reason, #pstate{connected = false}) ->
     ok;
-shutdown(conflict, _PState) ->
+terminate(conflict, _PState) ->
     ok;
-shutdown(discard, _PState) ->
+terminate(discard, _PState) ->
     ok;
-shutdown(Reason, PState) ->
+terminate(Reason, PState) ->
     ?LOG(info, "Shutdown for ~p", [Reason]),
     emqx_hooks:run('client.disconnected', [credentials(PState), Reason]).
 

src/emqx_rpc.erl

@@ -1,4 +1,4 @@
-%% Copyright (c) 2013-2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -28,4 +28,3 @@ multicall(Nodes, Mod, Fun, Args) ->
 
 cast(Node, Mod, Fun, Args) ->
     ?RPC:cast(Node, Mod, Fun, Args).
-

src/emqx_session.erl

@@ -71,8 +71,11 @@
           %% Clean Start Flag
           clean_start = false :: boolean(),
 
-          %% Client Binding: local | remote
+          %% Conn Binding: local | remote
-          binding = local :: local | remote,
+          %% binding = local :: local | remote,
+
+          %% Deliver fun
+          deliver_fun :: function(),
 
           %% ClientId: Identifier of Session
           client_id :: binary(),
@@ -157,6 +160,8 @@
 
 -export_type([attr/0]).
 
+-define(DEFAULT_BATCH_N, 1000).
+
 %% @doc Start a session proc.
 -spec(start_link(SessAttrs :: map()) -> {ok, pid()}).
 start_link(SessAttrs) ->
@@ -196,13 +201,13 @@ attrs(SPid) when is_pid(SPid) ->
     gen_server:call(SPid, attrs, infinity);
 
 attrs(#state{clean_start = CleanStart,
-             binding = Binding,
              client_id = ClientId,
+             conn_pid  = ConnPid,
              username = Username,
              expiry_interval = ExpiryInterval,
              created_at = CreatedAt}) ->
     [{clean_start, CleanStart},
-     {binding, Binding},
+     {binding, binding(ConnPid)},
      {client_id, ClientId},
      {username, Username},
      {expiry_interval, ExpiryInterval div 1000},
@@ -249,19 +254,19 @@ subscribe(SPid, PacketId, Properties, TopicFilters) ->
 
 %% @doc Called by connection processes when publishing messages
 -spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message())
-      -> emqx_types:deliver_results() | {error, term()}).
+      -> {ok, emqx_types:deliver_results()} | {error, term()}).
 publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) ->
     %% Publish QoS0 message directly
-    emqx_broker:publish(Msg);
+    {ok, emqx_broker:publish(Msg)};
 
 publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) ->
     %% Publish QoS1 message directly
-    emqx_broker:publish(Msg);
+    {ok, emqx_broker:publish(Msg)};
 
 publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) ->
     %% Register QoS2 message packet ID (and timestamp) to session, then publish
     case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of
-        ok -> emqx_broker:publish(Msg);
+        ok -> {ok, emqx_broker:publish(Msg)};
         {error, Reason} -> {error, Reason}
     end.
 
@@ -342,7 +347,7 @@ init([Parent, #{zone            := Zone,
     IdleTimout = get_env(Zone, idle_timeout, 30000),
     State = #state{idle_timeout      = IdleTimout,
                    clean_start       = CleanStart,
-                   binding           = binding(ConnPid),
+                   deliver_fun       = deliver_fun(ConnPid),
                    client_id         = ClientId,
                    username          = Username,
                    conn_pid          = ConnPid,
@@ -376,9 +381,18 @@ init_mqueue(Zone) ->
                        default_priority => get_env(Zone, mqueue_default_priority)
                       }).
 
+binding(undefined) -> undefined;
 binding(ConnPid) ->
     case node(ConnPid) =:= node() of true -> local; false -> remote end.
 
+deliver_fun(ConnPid) when node(ConnPid) == node() ->
+    fun(Packet) -> ConnPid ! {deliver, Packet}, ok end;
+deliver_fun(ConnPid) ->
+    Node = node(ConnPid),
+    fun(Packet) ->
+            emqx_rpc:cast(Node, erlang, send, [ConnPid, {deliver, Packet}])
+    end.
+
 handle_call(info, _From, State) ->
     reply(info(State), State);
 
@@ -539,7 +553,7 @@ handle_cast({resume, #{conn_pid        := ConnPid,
     true = link(ConnPid),
 
     State1 = State#state{conn_pid         = ConnPid,
-                         binding          = binding(ConnPid),
+                         deliver_fun      = deliver_fun(ConnPid),
                          old_conn_pid     = OldConnPid,
                          clean_start      = false,
                          retry_timer      = undefined,
@@ -566,25 +580,11 @@ handle_cast(Msg, State) ->
     emqx_logger:error("[Session] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-%% Batch dispatch
+handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, message) ->
-handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) ->
+    handle_dispatch([{Topic, Msg}], State);
-    noreply(lists:foldl(
-              fun(Msg, St) ->
-                  element(2, handle_info({dispatch, Topic, Msg}, St))
-              end, State, Msgs));
 
-%% Dispatch message
+handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) ->
-handle_info({dispatch, Topic, Msg = #message{}}, State) ->
+    handle_dispatch([{Topic, Msg} || Msg <- Msgs], State);
-    case emqx_shared_sub:is_ack_required(Msg) andalso not has_connection(State) of
-        true ->
-            %% Require ack, but we do not have connection
-            %% negative ack the message so it can try the next subscriber in the group
-            ok = emqx_shared_sub:nack_no_connection(Msg),
-            {noreply, State};
-        false ->
-            NewState = handle_dispatch(Topic, Msg, State),
-            noreply(ensure_stats_timer(maybe_gc({1, msg_size(Msg)}, NewState)))
-    end;
 
 %% Do nothing if the client has been disconnected.
 handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) ->
@@ -684,18 +684,11 @@ maybe_shutdown(Pid, Reason) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-has_connection(#state{conn_pid = Pid}) ->
+is_connection_alive(#state{conn_pid = Pid}) ->
     is_pid(Pid) andalso is_process_alive(Pid).
 
-handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) ->
+%%------------------------------------------------------------------------------
-    case maps:find(Topic, SubMap) of
+%% Suback and unsuback
-        {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} ->
-            run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State);
-        {ok, #{nl := Nl, qos := QoS, rap := Rap}} ->
-            run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State);
-        error ->
-            dispatch(emqx_message:unset_flag(dup, Msg), State)
-    end.
 
 suback(_From, undefined, _ReasonCodes) ->
     ignore;
@@ -722,7 +715,6 @@ kick(ClientId, OldConnPid, ConnPid) ->
 
 %%------------------------------------------------------------------------------
 %% Replay or Retry Delivery
-%%------------------------------------------------------------------------------
 
 %% Redeliver at once if force is true
 retry_delivery(Force, State = #state{inflight = Inflight}) ->
@@ -766,6 +758,7 @@ retry_delivery(Force, [{Type, Msg0, Ts} | Msgs], Now,
 %%------------------------------------------------------------------------------
 %% Send Will Message
 %%------------------------------------------------------------------------------
+
 send_willmsg(undefined) ->
     ignore;
 send_willmsg(WillMsg) ->
@@ -801,64 +794,156 @@ expire_awaiting_rel([{PacketId, Ts} | More], Now,
 
 is_awaiting_full(#state{max_awaiting_rel = 0}) ->
     false;
-is_awaiting_full(#state{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLen}) ->
+is_awaiting_full(#state{awaiting_rel = AwaitingRel,
+                        max_awaiting_rel = MaxLen}) ->
     maps:size(AwaitingRel) >= MaxLen.
 
 %%------------------------------------------------------------------------------
-%% Dispatch Messages
+%% Dispatch messages
 %%------------------------------------------------------------------------------
 
-run_dispatch_steps([], Msg, State) ->
+handle_dispatch(Msgs, State = #state{inflight = Inflight, subscriptions = SubMap}) ->
-    dispatch(Msg, State);
+    %% Drain the mailbox and batch deliver
-run_dispatch_steps([{nl, 1}|_Steps], #message{from = ClientId}, State = #state{client_id = ClientId}) ->
+    Msgs1 = drain_m(batch_n(Inflight), Msgs),
-    State;
+    %% Ack the messages for shared subscription
-run_dispatch_steps([{nl, _}|Steps], Msg, State) ->
+    Msgs2 = maybe_ack_shared(Msgs1, State),
-    run_dispatch_steps(Steps, Msg, State);
+    %% Process suboptions
-run_dispatch_steps([{qos, SubQoS}|Steps], Msg0 = #message{qos = PubQoS}, State = #state{upgrade_qos = false}) ->
+    Msgs3 = lists:foldr(
-    %% Ack immediately if a shared dispatch QoS is downgraded to 0
+              fun({Topic, Msg}, Acc) ->
-    Msg = case SubQoS =:= ?QOS_0 of
+                      SubOpts = find_subopts(Topic, SubMap),
-              true -> emqx_shared_sub:maybe_ack(Msg0);
+                      case process_subopts(SubOpts, Msg, State) of
-              false -> Msg0
+                          {ok, Msg1} -> [Msg1|Acc];
-          end,
+                          ignore -> Acc
-    run_dispatch_steps(Steps, Msg#message{qos = min(SubQoS, PubQoS)}, State);
+                      end
-run_dispatch_steps([{qos, SubQoS}|Steps], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) ->
+              end, [], Msgs2),
-    run_dispatch_steps(Steps, Msg#message{qos = max(SubQoS, PubQoS)}, State);
+    NState = batch_process(Msgs3, State),
-run_dispatch_steps([{rap, _Rap}|Steps], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) ->
+    noreply(ensure_stats_timer(NState)).
-    run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, true, Flags)}, State);
+
-run_dispatch_steps([{rap, 0}|Steps], Msg = #message{flags = Flags}, State = #state{}) ->
+batch_n(Inflight) ->
-    run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, false, Flags)}, State);
+    case emqx_inflight:max_size(Inflight) of
-run_dispatch_steps([{rap, _}|Steps], Msg, State) ->
+        0 -> ?DEFAULT_BATCH_N;
-    run_dispatch_steps(Steps, Msg, State);
+        Sz -> Sz - emqx_inflight:size(Inflight)
-run_dispatch_steps([{subid, SubId}|Steps], Msg, State) ->
+    end.
-    run_dispatch_steps(Steps, emqx_message:set_header('Subscription-Identifier', SubId, Msg), State).
+
+drain_m(Cnt, Msgs) when Cnt =< 0 ->
+    lists:reverse(Msgs);
+drain_m(Cnt, Msgs) ->
+    receive
+        {dispatch, Topic, Msg} ->
+            drain_m(Cnt-1, [{Topic, Msg}|Msgs])
+    after 0 ->
+        lists:reverse(Msgs)
+    end.
+
+%% Ack or nack the messages of shared subscription?
+maybe_ack_shared(Msgs, State) when is_list(Msgs) ->
+    lists:foldr(
+      fun({Topic, Msg}, Acc) ->
+            case maybe_ack_shared(Msg, State) of
+                ok -> Acc;
+                Msg1 -> [{Topic, Msg1}|Acc]
+            end
+      end, [], Msgs);
+
+maybe_ack_shared(Msg, State) ->
+    case emqx_shared_sub:is_ack_required(Msg) of
+        true -> do_ack_shared(Msg, State);
+        false -> Msg
+    end.
+
+do_ack_shared(Msg, State = #state{inflight = Inflight}) ->
+    case {is_connection_alive(State),
+          emqx_inflight:is_full(Inflight)} of
+        {false, _} ->
+            %% Require ack, but we do not have connection
+            %% negative ack the message so it can try the next subscriber in the group
+            emqx_shared_sub:nack_no_connection(Msg);
+        {_, true} ->
+            emqx_shared_sub:maybe_nack_dropped(Msg);
+         _ ->
+            %% Ack QoS1/QoS2 messages when message is delivered to connection.
+            %% NOTE: NOT to wait for PUBACK because:
+            %% The sender is monitoring this session process,
+            %% if the message is delivered to client but connection or session crashes,
+            %% sender will try to dispatch the message to the next shared subscriber.
+            %% This violates spec as QoS2 messages are not allowed to be sent to more
+            %% than one member in the group.
+            emqx_shared_sub:maybe_ack(Msg)
+    end.
+
+process_subopts([], Msg, _State) ->
+    {ok, Msg};
+process_subopts([{nl, 1}|_Opts], #message{from = ClientId}, #state{client_id = ClientId}) ->
+    ignore;
+process_subopts([{nl, _}|Opts], Msg, State) ->
+    process_subopts(Opts, Msg, State);
+process_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = false}) ->
+    process_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, State);
+process_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) ->
+    process_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, State);
+process_subopts([{rap, _Rap}|Opts], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) ->
+    process_subopts(Opts, Msg#message{flags = maps:put(retain, true, Flags)}, State);
+process_subopts([{rap, 0}|Opts], Msg = #message{flags = Flags}, State = #state{}) ->
+    process_subopts(Opts, Msg#message{flags = maps:put(retain, false, Flags)}, State);
+process_subopts([{rap, _}|Opts], Msg, State) ->
+    process_subopts(Opts, Msg, State);
+process_subopts([{subid, SubId}|Opts], Msg, State) ->
+    process_subopts(Opts, emqx_message:set_header('Subscription-Identifier', SubId, Msg), State).
+
+find_subopts(Topic, SubMap) ->
+    case maps:find(Topic, SubMap) of
+        {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} ->
+            [{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}];
+        {ok, #{nl := Nl, qos := QoS, rap := Rap}} ->
+            [{nl, Nl}, {qos, QoS}, {rap, Rap}];
+        error -> []
+    end.
+
+batch_process(Msgs, State) ->
+    {ok, Publishes, NState} = process_msgs(Msgs, [], State),
+    ok = batch_deliver(Publishes, NState),
+    maybe_gc(msg_cnt(Msgs), NState).
+
+process_msgs([], Publishes, State) ->
+    {ok, lists:reverse(Publishes), State};
+
+process_msgs([Msg|Msgs], Publishes, State) ->
+    case process_msg(Msg, State) of
+        {ok, Publish, NState} ->
+            process_msgs(Msgs, [Publish|Publishes], NState);
+        {ignore, NState} ->
+            process_msgs(Msgs, Publishes, NState)
+    end.
 
 %% Enqueue message if the client has been disconnected
-dispatch(Msg, State = #state{client_id = ClientId, username = Username, conn_pid = undefined}) ->
+process_msg(Msg, State = #state{conn_pid = undefined}) ->
-    case emqx_hooks:run('message.dropped', [#{client_id => ClientId, username => Username}, Msg]) of
+    {ignore, enqueue_msg(Msg, State)};
-        ok -> enqueue_msg(Msg, State);
-        stop -> State
-    end;
 
-%% Deliver qos0 message directly to client
+%% Prepare the qos0 message delivery
-dispatch(Msg = #message{qos = ?QOS_0} = Msg, State) ->
+process_msg(Msg = #message{qos = ?QOS_0}, State) ->
-    ok = deliver(undefined, Msg, State),
+    {ok, {publish, undefined, Msg}, State};
-    State;
 
-dispatch(Msg = #message{qos = QoS} = Msg,
+process_msg(Msg = #message{qos = QoS},
-         State = #state{next_pkt_id = PacketId, inflight = Inflight})
+            State = #state{next_pkt_id = PacketId, inflight = Inflight})
     when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 ->
     case emqx_inflight:is_full(Inflight) of
         true ->
-            enqueue_msg(Msg, State);
+            {ignore, enqueue_msg(Msg, State)};
         false ->
-            ok = deliver(PacketId, Msg, State),
+            Publish = {publish, PacketId, Msg},
-            await(PacketId, Msg, next_pkt_id(State))
+            NState = await(PacketId, Msg, State),
+            {ok, Publish, next_pkt_id(NState)}
     end.
 
-enqueue_msg(Msg, State = #state{mqueue = Q}) ->
+enqueue_msg(Msg, State = #state{mqueue = Q, client_id = ClientId, username = Username}) ->
     emqx_pd:update_counter(enqueue_stats, 1),
     {Dropped, NewQ} = emqx_mqueue:in(Msg, Q),
-    Dropped =/= undefined andalso emqx_shared_sub:maybe_nack_dropped(Dropped),
+    if
+        Dropped =/= undefined ->
+            SessProps = #{client_id => ClientId, username => Username},
+            emqx_hooks:run('message.dropped', [SessProps, Msg]);
+        true -> ok
+    end,
     State#state{mqueue = NewQ}.
 
 %%------------------------------------------------------------------------------
@@ -866,28 +951,22 @@ enqueue_msg(Msg, State = #state{mqueue = Q}) ->
 %%------------------------------------------------------------------------------
 
 redeliver({PacketId, Msg = #message{qos = QoS}}, State) ->
-    deliver(PacketId, if QoS =:= ?QOS_2 -> Msg;
+    Msg1 = if
-                         true -> emqx_message:set_flag(dup, Msg)
+               QoS =:= ?QOS_2 -> Msg;
-                      end, State);
+               true -> emqx_message:set_flag(dup, Msg)
+           end,
+    do_deliver(PacketId, Msg1, State);
 
-redeliver({pubrel, PacketId}, #state{conn_pid = ConnPid}) ->
+redeliver({pubrel, PacketId}, #state{deliver_fun = DeliverFun}) ->
-    ConnPid ! {deliver, {pubrel, PacketId}}.
+    DeliverFun({pubrel, PacketId}).
 
-deliver(PacketId, Msg, State) ->
+do_deliver(PacketId, Msg, #state{deliver_fun = DeliverFun}) ->
     emqx_pd:update_counter(deliver_stats, 1),
-    %% Ack QoS1/QoS2 messages when message is delivered to connection.
+    DeliverFun({publish, PacketId, Msg}).
-    %% NOTE: NOT to wait for PUBACK because:
-    %% The sender is monitoring this session process,
-    %% if the message is delivered to client but connection or session crashes,
-    %% sender will try to dispatch the message to the next shared subscriber.
-    %% This violates spec as QoS2 messages are not allowed to be sent to more
-    %% than one member in the group.
-    do_deliver(PacketId, emqx_shared_sub:maybe_ack(Msg), State).
 
-do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = local}) ->
+batch_deliver(Publishes, #state{deliver_fun = DeliverFun}) ->
-    ConnPid ! {deliver, {publish, PacketId, Msg}}, ok;
+    emqx_pd:update_counter(deliver_stats, length(Publishes)),
-do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = remote}) ->
+    DeliverFun(Publishes).
-    emqx_rpc:cast(node(ConnPid), erlang, send, [ConnPid, {deliver, {publish, PacketId, Msg}}]).
 
 %%------------------------------------------------------------------------------
 %% Awaiting ACK for QoS1/QoS2 Messages
@@ -932,26 +1011,31 @@ acked(pubcomp, PacketId, State = #state{inflight = Inflight}) ->
 dequeue(State = #state{conn_pid = undefined}) ->
     State;
 
-dequeue(State = #state{inflight = Inflight}) ->
+dequeue(State = #state{inflight = Inflight, mqueue = Q}) ->
-    case emqx_inflight:is_full(Inflight) of
+    case emqx_mqueue:is_empty(Q)
-        true  -> State;
+         orelse emqx_inflight:is_full(Inflight) of
-        false -> dequeue2(State)
+        true -> State;
+        false ->
+            {Msgs, Q1} = drain_q(batch_n(Inflight), [], Q),
+            batch_process(lists:reverse(Msgs), State#state{mqueue = Q1})
     end.
 
-dequeue2(State = #state{mqueue = Q}) ->
+drain_q(Cnt, Msgs, Q) when Cnt =< 0 ->
+    {Msgs, Q};
+
+drain_q(Cnt, Msgs, Q) ->
     case emqx_mqueue:out(Q) of
-        {empty, _Q} -> State;
+        {empty, _Q} -> {Msgs, Q};
         {{value, Msg}, Q1} ->
-            %% Dequeue more
+            drain_q(Cnt-1, [Msg|Msgs], Q1)
-            dequeue(dispatch(Msg, State#state{mqueue = Q1}))
     end.
 
 %%------------------------------------------------------------------------------
 %% Ensure timers
 
-ensure_await_rel_timer(State = #state{await_rel_timer = undefined, await_rel_timeout = Timeout}) ->
+ensure_await_rel_timer(State = #state{await_rel_timer = undefined,
+                                      await_rel_timeout = Timeout}) ->
     ensure_await_rel_timer(Timeout, State);
-
 ensure_await_rel_timer(State) ->
     State.
 
@@ -960,7 +1044,8 @@ ensure_await_rel_timer(Timeout, State = #state{await_rel_timer = undefined}) ->
 ensure_await_rel_timer(_Timeout, State) ->
     State.
 
-ensure_retry_timer(State = #state{retry_timer = undefined, retry_interval = Interval}) ->
+ensure_retry_timer(State = #state{retry_timer = undefined,
+                                  retry_interval = Interval}) ->
     ensure_retry_timer(Interval, State);
 ensure_retry_timer(State) ->
     State.
@@ -970,7 +1055,8 @@ ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) ->
 ensure_retry_timer(_Timeout, State) ->
     State.
 
-ensure_expire_timer(State = #state{expiry_interval = Interval}) when Interval > 0 andalso Interval =/= 16#ffffffff ->
+ensure_expire_timer(State = #state{expiry_interval = Interval})
+  when Interval > 0 andalso Interval =/= 16#ffffffff ->
     State#state{expiry_timer = emqx_misc:start_timer(Interval * 1000, expired)};
 ensure_expire_timer(State) ->
     State.
@@ -997,15 +1083,20 @@ next_pkt_id(State = #state{next_pkt_id = 16#FFFF}) ->
 next_pkt_id(State = #state{next_pkt_id = Id}) ->
     State#state{next_pkt_id = Id + 1}.
 
+%%------------------------------------------------------------------------------
+%% Maybe GC
+
+msg_cnt(Msgs) ->
+    lists:foldl(fun(Msg, {Cnt, Oct}) ->
+                        {Cnt+1, Oct+msg_size(Msg)}
+                end, {0, 0}, Msgs).
+
 %% Take only the payload size into account, add other fields if necessary
 msg_size(#message{payload = Payload}) -> payload_size(Payload).
 
 %% Payload should be binary(), but not 100% sure. Need dialyzer!
 payload_size(Payload) -> erlang:iolist_size(Payload).
 
-%%------------------------------------------------------------------------------
-%% Maybe GC
-
 maybe_gc(_, State = #state{gc_state = undefined}) ->
     State;
 maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) ->

src/emqx_sup.erl

@@ -61,9 +61,6 @@ init([]) ->
     RouterSup = supervisor_spec(emqx_router_sup),
     %% Broker Sup
     BrokerSup = supervisor_spec(emqx_broker_sup),
-    %% BridgeSup
-    LocalBridgeSup = supervisor_spec(emqx_local_bridge_sup_sup),
-
     BridgeSup = supervisor_spec(emqx_bridge_sup),
     %% AccessControl
     AccessControl = worker_spec(emqx_access_control),
@@ -77,7 +74,6 @@ init([]) ->
           [KernelSup,
            RouterSup,
            BrokerSup,
-           LocalBridgeSup,
            BridgeSup,
            AccessControl,
            SMSup,
@@ -92,4 +88,3 @@ worker_spec(M) ->
     {M, {M, start_link, []}, permanent, 30000, worker, [M]}.
 supervisor_spec(M) ->
     {M, {M, start_link, []}, permanent, infinity, supervisor, [M]}.
-

src/emqx_sys_mon.erl

@@ -163,5 +163,6 @@ safe_publish(Event, WarnMsg) ->
     emqx_broker:safe_publish(sysmon_msg(Topic, iolist_to_binary(WarnMsg))).
 
 sysmon_msg(Topic, Payload) ->
-    emqx_message:make(?SYSMON, #{sys => true}, Topic, Payload).
+    Msg = emqx_message:make(?SYSMON, Topic, Payload),
+    emqx_message:set_flag(sys, Msg).
 

src/emqx_sys_sup.erl

@@ -24,17 +24,23 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
-    Sys = #{id       => sys,
+    {ok, {{one_for_one, 10, 100}, [child_spec(emqx_sys, worker),
-            start    => {emqx_sys, start_link, []},
+                                   child_spec(emqx_sys_mon, worker, [emqx_config:get_env(sysmon, [])]),
-            restart  => permanent,
+                                   child_spec(emqx_os_mon, worker, [emqx_config:get_env(os_mon, [])]),
-            shutdown => 5000,
+                                   child_spec(emqx_vm_mon, worker, [emqx_config:get_env(vm_mon, [])])]}}.
-            type     => worker,
+
-            modules  => [emqx_sys]},
+%%--------------------------------------------------------------------
-    Sysmon = #{id       => sys_mon,
+%% Internal functions
-               start    => {emqx_sys_mon, start_link, [emqx_config:get_env(sysmon, [])]},
+%%--------------------------------------------------------------------
-               restart  => permanent,
+
-               shutdown => 5000,
+child_spec(M, worker) ->
-               type     => worker,
+    child_spec(M, worker, []).
-               modules  => [emqx_sys_mon]},
+
-    {ok, {{one_for_one, 10, 100}, [Sys, Sysmon]}}.
+child_spec(M, worker, A) ->
+    #{id       => M,
+      start    => {M, start_link, A},
+      restart  => permanent,
+      shutdown => 5000,
+      type     => worker,
+      modules  => [M]}.
 

src/emqx_topic.erl

@@ -20,7 +20,7 @@
 -export([triples/1]).
 -export([words/1]).
 -export([wildcard/1]).
--export([join/1]).
+-export([join/1, prepend/2]).
 -export([feed_var/3]).
 -export([systop/1]).
 -export([parse/1, parse/2]).
@@ -129,10 +129,23 @@ join(root, W) ->
 join(Parent, W) ->
     <<(bin(Parent))/binary, $/, (bin(W))/binary>>.
 
+%% @doc Prepend a topic prefix.
+%% Ensured to have only one / between prefix and suffix.
+prepend(root, W) -> bin(W);
+prepend(undefined, W) -> bin(W);
+prepend(<<>>, W) -> bin(W);
+prepend(Parent0, W) ->
+    Parent = bin(Parent0),
+    case binary:last(Parent) of
+        $/ -> <<Parent/binary, (bin(W))/binary>>;
+        _ -> join(Parent, W)
+    end.
+
 bin('')  -> <<>>;
 bin('+') -> <<"+">>;
 bin('#') -> <<"#">>;
-bin(B) when is_binary(B) -> B.
+bin(B) when is_binary(B) -> B;
+bin(L) when is_list(L) -> list_to_binary(L).
 
 levels(Topic) when is_binary(Topic) ->
     length(words(Topic)).

src/emqx_types.erl

@@ -31,7 +31,7 @@
 -type(pubsub() :: publish | subscribe).
 -type(topic() :: binary()).
 -type(subid() :: binary() | atom()).
--type(subopts() :: #{qos    := integer(),
+-type(subopts() :: #{qos    := emqx_mqtt_types:qos(),
                      share  => binary(),
                      atom() => term()
                     }).
@@ -59,4 +59,3 @@
 -type(alarm() :: #alarm{}).
 -type(plugin() :: #plugin{}).
 -type(command() :: #command{}).
-

src/emqx_vm_mon.erl

@@ -0,0 +1,118 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_vm_mon).
+
+-behaviour(gen_server).
+
+-export([start_link/1]).
+
+-export([init/1,
+         handle_call/3,
+         handle_cast/2,
+         handle_info/2,
+         terminate/2,
+         code_change/3]).
+
+-export([get_check_interval/0,
+         set_check_interval/1,
+         get_process_high_watermark/0,
+         set_process_high_watermark/1,
+         get_process_low_watermark/0,
+         set_process_low_watermark/1]).
+
+-define(VM_MON, ?MODULE).
+
+%%----------------------------------------------------------------------
+%% API
+%%----------------------------------------------------------------------
+
+start_link(Opts) ->
+    gen_server:start_link({local, ?VM_MON}, ?MODULE, [Opts], []).
+
+get_check_interval() ->
+    call(get_check_interval).
+
+set_check_interval(Seconds) ->
+    call({set_check_interval, Seconds}).
+
+get_process_high_watermark() ->
+    call(get_process_high_watermark).
+
+set_process_high_watermark(Float) ->
+    call({set_process_high_watermark, Float}).
+
+get_process_low_watermark() ->
+    call(get_process_low_watermark).
+
+set_process_low_watermark(Float) ->
+    call({set_process_low_watermark, Float}).
+
+%%----------------------------------------------------------------------
+%% gen_server callbacks
+%%----------------------------------------------------------------------
+
+init([Opts]) ->
+    {ok, ensure_check_timer(#{check_interval => proplists:get_value(check_interval, Opts, 30),
+                              process_high_watermark => proplists:get_value(process_high_watermark, Opts, 0.70),
+                              process_low_watermark => proplists:get_value(process_low_watermark, Opts, 0.50),
+                              timer => undefined})}.
+
+handle_call(get_check_interval, _From, State) ->
+    {reply, maps:get(check_interval, State, undefined), State};
+handle_call({set_check_interval, Seconds}, _From, State) ->
+    {reply, ok, State#{check_interval := Seconds}};
+
+handle_call(get_process_high_watermark, _From, State) ->
+    {reply, maps:get(process_high_watermark, State, undefined), State};
+handle_call({set_process_high_watermark, Float}, _From, State) ->
+    {reply, ok, State#{process_high_watermark := Float}};
+
+handle_call(get_process_low_watermark, _From, State) ->
+    {reply, maps:get(process_low_watermark, State, undefined), State};
+handle_call({set_process_low_watermark, Float}, _From, State) ->
+    {reply, ok, State#{process_low_watermark := Float}};
+
+handle_call(_Request, _From, State) ->
+    {reply, ok, State}.
+
+handle_cast(_Request, State) ->
+    {noreply, State}.
+
+handle_info({timeout, Timer, check}, State = #{timer := Timer,
+                                               process_high_watermark := ProcHighWatermark,
+                                               process_low_watermark := ProcLowWatermark}) ->
+    ProcessCount = erlang:system_info(process_count),
+    case ProcessCount / erlang:system_info(process_limit) of
+        Percent when Percent >= ProcHighWatermark ->
+            alarm_handler:set_alarm({too_many_processes, ProcessCount});
+        Percent when Percent < ProcLowWatermark ->
+            alarm_handler:clear_alarm(too_many_processes)
+    end,
+    {noreply, ensure_check_timer(State)}.
+
+terminate(_Reason, #{timer := Timer}) ->
+    emqx_misc:cancel_timer(Timer).
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%----------------------------------------------------------------------
+%% Internal functions
+%%----------------------------------------------------------------------
+call(Req) ->
+    gen_server:call(?VM_MON, Req, infinity).
+
+ensure_check_timer(State = #{check_interval := Interval}) ->
+    State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}.

src/emqx_ws_connection.erl

@@ -18,7 +18,8 @@
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
 
--export([info/1, attrs/1]).
+-export([info/1]).
+-export([attrs/1]).
 -export([stats/1]).
 -export([kick/1]).
 -export([session/1]).
@@ -37,7 +38,7 @@
           sockname,
           idle_timeout,
           proto_state,
-          parser_state,
+          parse_state,
           keepalive,
           enable_stats,
           stats_timer,
@@ -127,25 +128,23 @@ websocket_init(#state{request = Req, options = Options}) ->
     ProtoState = emqx_protocol:init(#{peername => Peername,
                                       sockname => Sockname,
                                       peercert => Peercert,
-                                      sendfun  => send_fun(self())}, Options),
+                                      sendfun  => send_fun(self()),
+                                      conn_mod => ?MODULE}, Options),
     ParserState = emqx_protocol:parser(ProtoState),
     Zone = proplists:get_value(zone, Options),
     EnableStats = emqx_zone:get_env(Zone, enable_stats, true),
     IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000),
-
     emqx_logger:set_metadata_peername(esockd_net:format(Peername)),
     {ok, #state{peername     = Peername,
                 sockname     = Sockname,
-                parser_state = ParserState,
+                parse_state  = ParserState,
                 proto_state  = ProtoState,
                 enable_stats = EnableStats,
                 idle_timeout = IdleTimout}}.
 
 send_fun(WsPid) ->
-    fun(Packet, Options) ->
+    fun(Data) ->
-        Data = emqx_frame:serialize(Packet, Options),
         BinSize = iolist_size(Data),
-        emqx_metrics:trans(inc, 'bytes/sent', BinSize),
         emqx_pd:update_counter(send_cnt, 1),
         emqx_pd:update_counter(send_oct, BinSize),
         WsPid ! {binary, iolist_to_binary(Data)},
@@ -159,15 +158,15 @@ websocket_handle({binary, <<>>}, State) ->
     {ok, ensure_stats_timer(State)};
 websocket_handle({binary, [<<>>]}, State) ->
     {ok, ensure_stats_timer(State)};
-websocket_handle({binary, Data}, State = #state{parser_state = ParserState,
+websocket_handle({binary, Data}, State = #state{parse_state = ParseState,
-                                                proto_state  = ProtoState}) ->
+                                                proto_state = ProtoState}) ->
     ?LOG(debug, "RECV ~p", [Data]),
     BinSize = iolist_size(Data),
     emqx_pd:update_counter(recv_oct, BinSize),
     emqx_metrics:trans(inc, 'bytes/received', BinSize),
-    try emqx_frame:parse(iolist_to_binary(Data), ParserState) of
+    try emqx_frame:parse(iolist_to_binary(Data), ParseState) of
-        {more, ParserState1} ->
+        {more, ParseState1} ->
-            {ok, State#state{parser_state = ParserState1}};
+            {ok, State#state{parse_state = ParseState1}};
         {ok, Packet, Rest} ->
             emqx_metrics:received(Packet),
             emqx_pd:update_counter(recv_cnt, 1),
@@ -248,10 +247,10 @@ websocket_info({keepalive, check}, State = #state{keepalive = KeepAlive}) ->
         {ok, KeepAlive1} ->
             {ok, State#state{keepalive = KeepAlive1}};
         {error, timeout} ->
-            ?LOG(debug, "Keepalive Timeout!", []),
+            ?LOG(debug, "Keepalive Timeout!"),
             shutdown(keepalive_timeout, State);
         {error, Error} ->
-            ?LOG(warning, "Keepalive error - ~p", [Error]),
+            ?LOG(error, "Keepalive error - ~p", [Error]),
             shutdown(keepalive_error, State)
     end;
 
@@ -277,15 +276,14 @@ terminate(SockError, _Req, #state{keepalive   = Keepalive,
                                   proto_state = ProtoState,
                                   shutdown    = Shutdown}) ->
 
-    ?LOG(debug, "Terminated for ~p, sockerror: ~p",
+    ?LOG(debug, "Terminated for ~p, sockerror: ~p", [Shutdown, SockError]),
-           [Shutdown, SockError]),
     emqx_keepalive:cancel(Keepalive),
     case {ProtoState, Shutdown} of
         {undefined, _} -> ok;
         {_, {shutdown, Reason}} ->
-            emqx_protocol:shutdown(Reason, ProtoState);
+            emqx_protocol:terminate(Reason, ProtoState);
         {_, Error} ->
-            emqx_protocol:shutdown(Error, ProtoState)
+            emqx_protocol:terminate(Error, ProtoState)
     end.
 
 %%------------------------------------------------------------------------------
@@ -293,7 +291,7 @@ terminate(SockError, _Req, #state{keepalive   = Keepalive,
 %%------------------------------------------------------------------------------
 
 reset_parser(State = #state{proto_state = ProtoState}) ->
-    State#state{parser_state = emqx_protocol:parser(ProtoState)}.
+    State#state{parse_state = emqx_protocol:parser(ProtoState)}.
 
 ensure_stats_timer(State = #state{enable_stats = true,
                                   stats_timer  = undefined,

test/emqx_alarm_handler_SUITE.erl

@@ -0,0 +1,145 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_alarm_handler_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-include_lib("common_test/include/ct.hrl").
+
+-include("emqx_mqtt.hrl").
+-include("emqx.hrl").
+
+all() -> [t_alarm_handler, t_logger_handler].
+
+init_per_suite(Config) ->
+    [start_apps(App, {SchemaFile, ConfigFile}) ||
+        {App, SchemaFile, ConfigFile}
+            <- [{emqx, local_path("priv/emqx.schema"),
+                       local_path("etc/emqx.conf")}]],
+    Config.
+
+end_per_suite(_Config) ->
+    application:stop(emqx).
+
+local_path(RelativePath) ->
+    filename:join([get_base_dir(), RelativePath]).
+
+get_base_dir() ->
+    {file, Here} = code:is_loaded(?MODULE),
+    filename:dirname(filename:dirname(Here)).
+
+start_apps(App, {SchemaFile, ConfigFile}) ->
+    read_schema_configs(App, {SchemaFile, ConfigFile}),
+    set_special_configs(App),
+    application:ensure_all_started(App).
+
+read_schema_configs(App, {SchemaFile, ConfigFile}) ->
+    ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]),
+    Schema = cuttlefish_schema:files([SchemaFile]),
+    Conf = conf_parse:file(ConfigFile),
+    NewConfig = cuttlefish_generator:map(Schema, Conf),
+    Vals = proplists:get_value(App, NewConfig, []),
+    [application:set_env(App, Par, Value) || {Par, Value} <- Vals].
+
+set_special_configs(_App) ->
+    ok.
+
+with_connection(DoFun) ->
+    {ok, Sock} = emqx_client_sock:connect({127, 0, 0, 1}, 1883,
+                                          [binary, {packet, raw}, {active, false}],
+                                          3000),
+    try
+        DoFun(Sock)
+    after
+        emqx_client_sock:close(Sock)
+    end.
+
+t_alarm_handler(_) ->
+    with_connection(
+        fun(Sock) ->
+            emqx_client_sock:send(Sock,
+                                  raw_send_serialize(
+                                      ?CONNECT_PACKET(
+                                          #mqtt_packet_connect{
+                                          proto_ver  = ?MQTT_PROTO_V5}),
+                                      #{version => ?MQTT_PROTO_V5}
+                                  )),
+            {ok, Data} = gen_tcp:recv(Sock, 0),
+            {ok, ?CONNACK_PACKET(?RC_SUCCESS), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5),
+
+            Topic1 = emqx_topic:systop(<<"alarms/alarm_for_test/alert">>),
+            Topic2 = emqx_topic:systop(<<"alarms/alarm_for_test/clear">>),
+            SubOpts = #{rh => 1, qos => ?QOS_2, rap => 0, nl => 0, rc => 0},
+            emqx_client_sock:send(Sock, 
+                                  raw_send_serialize(
+                                      ?SUBSCRIBE_PACKET(
+                                          1, 
+                                          [{Topic1, SubOpts},
+                                           {Topic2, SubOpts}]), 
+                                      #{version => ?MQTT_PROTO_V5})),
+
+            {ok, Data2} = gen_tcp:recv(Sock, 0),
+            {ok, ?SUBACK_PACKET(1, #{}, [2, 2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5),
+
+            alarm_handler:set_alarm({alarm_for_test, #alarm{id = alarm_for_test,
+                                                            severity = error,
+                                                            title="alarm title",
+                                                            summary="alarm summary"}}),
+
+            {ok, Data3} = gen_tcp:recv(Sock, 0),
+
+            {ok, ?PUBLISH_PACKET(?QOS_0, Topic1, _, _), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5),
+
+            ?assertEqual(true, lists:keymember(alarm_for_test, 1, emqx_alarm_handler:get_alarms())),
+
+            alarm_handler:clear_alarm(alarm_for_test),
+
+            {ok, Data4} = gen_tcp:recv(Sock, 0),
+
+            {ok, ?PUBLISH_PACKET(?QOS_0, Topic2, _, _), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5),
+
+            ?assertEqual(false, lists:keymember(alarm_for_test, 1, emqx_alarm_handler:get_alarms()))
+
+        end).
+
+t_logger_handler(_) ->
+    %% Meck supervisor report
+    logger:log(error, #{label => {supervisor, start_error}, 
+                        report => [{supervisor, {local, tmp_sup}}, 
+                                   {errorContext, shutdown}, 
+                                   {reason, reached_max_restart_intensity}, 
+                                   {offender, [{pid, meck},
+                                               {id, meck},
+                                               {mfargs, {meck, start_link, []}},
+                                               {restart_type, permanent},
+                                               {shutdown, 5000},
+                                               {child_type, worker}]}]}, 
+               #{logger_formatter => #{title => "SUPERVISOR REPORT"},
+                 report_cb => fun logger:format_otp_report/1}),
+    ?assertEqual(true, lists:keymember(supervisor_report, 1, emqx_alarm_handler:get_alarms())).
+
+raw_send_serialize(Packet) ->
+    emqx_frame:serialize(Packet).
+
+raw_send_serialize(Packet, Opts) ->
+    emqx_frame:serialize(Packet, Opts).
+
+raw_recv_parse(P, ProtoVersion) ->
+    emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE,
+                                 version         => ProtoVersion}}).
+

test/emqx_bridge_SUITE.erl

@@ -14,45 +14,183 @@
 
 -module(emqx_bridge_SUITE).
 
--compile(export_all).
+-export([all/0, init_per_suite/1, end_per_suite/1]).
--compile(nowarn_export_all).
+-export([t_rpc/1,
+         t_mqtt/1,
+         t_mngr/1
+        ]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include("emqx_mqtt.hrl").
+-include("emqx.hrl").
 
-all() ->
+-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)).
-    [bridge_test].
+
+all() -> [t_rpc,
+          t_mqtt,
+          t_mngr].
 
 init_per_suite(Config) ->
-    emqx_ct_broker_helpers:run_setup_steps(),
+    case node() of
-    Config.
+        nonode@nohost ->
+            net_kernel:start(['emqx@127.0.0.1', longnames]);
+        _ ->
+            ok
+    end,
+    emqx_ct_broker_helpers:run_setup_steps([{log_level, error} | Config]).
 
 end_per_suite(_Config) ->
     emqx_ct_broker_helpers:run_teardown_steps().
 
-bridge_test(_) ->
+t_mngr(Config) when is_list(Config) ->
-    #{msg := <<"start bridge successfully">>}
+    Subs = [{<<"a">>, 1}, {<<"b">>, 2}],
-        = emqx_bridge:start_bridge(aws),
+    Cfg = #{address => node(),
-    test_forwards(),
+            forwards => [<<"mngr">>],
-    test_subscriptions(0),
+            connect_module => emqx_bridge_rpc,
-    test_subscriptions(1),
+            mountpoint => <<"forwarded">>,
-    test_subscriptions(2),
+            subscriptions => Subs,
-    #{msg := <<"stop bridge successfully">>}
+            start_type => auto
-        = emqx_bridge:stop_bridge(aws),
+           },
+    Name = ?FUNCTION_NAME,
+    {ok, Pid} = emqx_bridge:start_link(Name, Cfg),
+    try
+        ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Name)),
+        ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr")),
+        ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr2")),
+        ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_bridge:get_forwards(Pid)),
+        ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr2")),
+        ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr3")),
+        ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Pid)),
+        ?assertEqual({error, no_remote_subscription_support},
+                     emqx_bridge:ensure_subscription_present(Pid, <<"t">>, 0)),
+        ?assertEqual({error, no_remote_subscription_support},
+                     emqx_bridge:ensure_subscription_absent(Pid, <<"t">>)),
+        ?assertEqual(Subs, emqx_bridge:get_subscriptions(Pid))
+    after
+        ok = emqx_bridge:stop(Pid)
+    end.
+
+%% A loopback RPC to local node
+t_rpc(Config) when is_list(Config) ->
+    Cfg = #{address => node(),
+            forwards => [<<"t_rpc/#">>],
+            connect_module => emqx_bridge_rpc,
+            mountpoint => <<"forwarded">>,
+            start_type => auto
+           },
+    {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg),
+    ClientId = <<"ClientId">>,
+    try
+        {ok, ConnPid} = emqx_mock_client:start_link(ClientId),
+        {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal),
+        %% message from a different client, to avoid getting terminated by no-local
+        Msg1 = emqx_message:make(<<"ClientId-2">>, ?QOS_2, <<"t_rpc/one">>, <<"hello">>),
+        ok = emqx_session:subscribe(SPid, [{<<"forwarded/t_rpc/one">>, #{qos => ?QOS_1}}]),
+        PacketId = 1,
+        emqx_session:publish(SPid, PacketId, Msg1),
+        ?wait(case emqx_mock_client:get_last_message(ConnPid) of
+                  [{publish, PacketId, #message{topic = <<"forwarded/t_rpc/one">>}}] ->
+                      true;
+                  Other ->
+                      Other
+              end, 4000),
+        emqx_mock_client:close_session(ConnPid)
+    after
+        ok = emqx_bridge:stop(Pid)
+    end.
+
+%% Full data loopback flow explained:
+%% test-pid --->  mock-cleint ----> local-broker ---(local-subscription)--->
+%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) -->
+%% bridge(import) --(mecked message sending)--> test-pid
+t_mqtt(Config) when is_list(Config) ->
+    SendToTopic = <<"t_mqtt/one">>,
+    SendToTopic2 = <<"t_mqtt/two">>,
+    Mountpoint = <<"forwarded/${node}/">>,
+    ForwardedTopic = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic]),
+    ForwardedTopic2 = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic2]),
+    Cfg = #{address => "127.0.0.1:1883",
+            forwards => [SendToTopic],
+            connect_module => emqx_bridge_mqtt,
+            mountpoint => Mountpoint,
+            username => "user",
+            clean_start => true,
+            client_id => "bridge_aws",
+            keepalive => 60000,
+            max_inflight => 32,
+            password => "passwd",
+            proto_ver => mqttv4,
+            queue => #{replayq_dir => "data/t_mqtt/",
+                       replayq_seg_bytes => 10000,
+                       batch_bytes_limit => 1000,
+                       batch_count_limit => 10
+                      },
+            reconnect_delay_ms => 1000,
+            ssl => false,
+            %% Consume back to forwarded message for verification
+            %% NOTE: this is a indefenite loopback without mocking emqx_bridge:import_batch/2
+            subscriptions => [{ForwardedTopic, _QoS = 1}],
+            start_type => auto
+           },
+    Tester = self(),
+    Ref = make_ref(),
+    meck:new(emqx_bridge, [passthrough, no_history]),
+    meck:expect(emqx_bridge, import_batch, 2,
+                fun(Batch, AckFun) ->
+                        Tester ! {Ref, Batch},
+                        AckFun()
+                end),
+    {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg),
+    ClientId = <<"client-1">>,
+    try
+        ?assertEqual([{ForwardedTopic, 1}], emqx_bridge:get_subscriptions(Pid)),
+        ok = emqx_bridge:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1),
+        ok = emqx_bridge:ensure_forward_present(Pid, SendToTopic2),
+        ?assertEqual([{ForwardedTopic, 1},
+                      {ForwardedTopic2, 1}], emqx_bridge:get_subscriptions(Pid)),
+        {ok, ConnPid} = emqx_mock_client:start_link(ClientId),
+        {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal),
+        %% message from a different client, to avoid getting terminated by no-local
+        Max = 100,
+        Msgs = lists:seq(1, Max),
+        lists:foreach(fun(I) ->
+                              Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic, integer_to_binary(I)),
+                              emqx_session:publish(SPid, I, Msg)
+                      end, Msgs),
+        ok = receive_and_match_messages(Ref, Msgs),
+        Msgs2 = lists:seq(Max + 1, Max * 2),
+        lists:foreach(fun(I) ->
+                              Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic2, integer_to_binary(I)),
+                              emqx_session:publish(SPid, I, Msg)
+                      end, Msgs2),
+        ok = receive_and_match_messages(Ref, Msgs2),
+        emqx_mock_client:close_session(ConnPid)
+    after
+        ok = emqx_bridge:stop(Pid),
+        meck:unload(emqx_bridge)
+    end.
+
+receive_and_match_messages(Ref, Msgs) ->
+    TRef = erlang:send_after(timer:seconds(5), self(), {Ref, timeout}),
+    try
+        do_receive_and_match_messages(Ref, Msgs)
+    after
+        erlang:cancel_timer(TRef)
+    end,
     ok.
 
-test_forwards() ->
+do_receive_and_match_messages(_Ref, []) -> ok;
-    emqx_bridge:add_forward(aws, <<"test_forwards">>),
+do_receive_and_match_messages(Ref, [I | Rest] = Exp) ->
-    [<<"test_forwards">>, <<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws),
+    receive
-    emqx_bridge:del_forward(aws, <<"test_forwards">>),
+        {Ref, timeout} -> erlang:error(timeout);
-    [<<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws),
+        {Ref, [#{payload := P} = Msg]} ->
-    ok.
+            case binary_to_integer(P) of
-
+                I ->  %% exact match
-test_subscriptions(QoS) ->
+                    do_receive_and_match_messages(Ref, Rest);
-    emqx_bridge:add_subscription(aws, <<"test_subscriptions">>, QoS),
+                J when J < I -> %% allow retry
-    [{<<"test_subscriptions">>, QoS},
+                    do_receive_and_match_messages(Ref, Exp);
-     {<<"cmd/topic1">>, 1},
+                _Other ->
-     {<<"cmd/topic2">>, 1}] = emqx_bridge:show_subscriptions(aws),
+                    throw({unexpected, Msg, Exp})
-    emqx_bridge:del_subscription(aws, <<"test_subscriptions">>),
+            end
-     [{<<"cmd/topic1">>,1}, {<<"cmd/topic2">>,1}] = emqx_bridge:show_subscriptions(aws),
+    end.
-    ok.

test/emqx_bridge_mqtt_tests.erl

@@ -0,0 +1,54 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_bridge_mqtt_tests).
+-include_lib("eunit/include/eunit.hrl").
+-include("emqx_mqtt.hrl").
+
+send_and_ack_test() ->
+    %% delegate from gen_rpc to rpc for unit test
+    meck:new(emqx_client, [passthrough, no_history]),
+    meck:expect(emqx_client, start_link, 1,
+                fun(#{msg_handler := Hdlr}) ->
+                        {ok, spawn_link(fun() -> fake_client(Hdlr) end)}
+                end),
+    meck:expect(emqx_client, connect, 1, {ok, dummy}),
+    meck:expect(emqx_client, stop, 1,
+                fun(Pid) -> Pid ! stop end),
+    meck:expect(emqx_client, publish, 2,
+                fun(Client, Msg) ->
+                        Client ! {publish, Msg},
+                        {ok, Msg} %% as packet id
+                end),
+    try
+        Max = 100,
+        Batch = lists:seq(1, Max),
+        {ok, Ref, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}),
+        %% return last packet id as batch reference
+        {ok, AckRef} = emqx_bridge_mqtt:send(Conn, Batch),
+        %% expect batch ack
+        receive {batch_ack, AckRef} -> ok end,
+        ok = emqx_bridge_mqtt:stop(Ref, Conn)
+    after
+        meck:unload(emqx_client)
+    end.
+
+fake_client(#{puback := PubAckCallback} = Hdlr) ->
+    receive
+        {publish, PktId} ->
+            PubAckCallback(#{packet_id => PktId, reason_code => ?RC_SUCCESS}),
+            fake_client(Hdlr);
+        stop ->
+            exit(normal)
+    end.

test/emqx_bridge_rpc_tests.erl

@@ -0,0 +1,43 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_bridge_rpc_tests).
+-include_lib("eunit/include/eunit.hrl").
+
+send_and_ack_test() ->
+    %% delegate from gen_rpc to rpc for unit test
+    meck:new(gen_rpc, [passthrough, no_history]),
+    meck:expect(gen_rpc, call, 4,
+                fun(Node, Module, Fun, Args) ->
+                        rpc:call(Node, Module, Fun, Args)
+                end),
+    meck:expect(gen_rpc, cast, 4,
+                fun(Node, Module, Fun, Args) ->
+                        rpc:cast(Node, Module, Fun, Args)
+                end),
+    meck:new(emqx_bridge, [passthrough, no_history]),
+    meck:expect(emqx_bridge, import_batch, 2,
+                fun(batch, AckFun) -> AckFun() end),
+    try
+        {ok, Pid, Node} = emqx_bridge_rpc:start(#{address => node()}),
+        {ok, Ref} = emqx_bridge_rpc:send(Node, batch),
+        receive
+            {batch_ack, Ref} ->
+                ok
+        end,
+        ok = emqx_bridge_rpc:stop(Pid, Node)
+    after
+        meck:unload(gen_rpc),
+        meck:unload(emqx_bridge)
+    end.

test/emqx_bridge_tests.erl

@@ -0,0 +1,157 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_bridge_tests).
+-behaviour(emqx_bridge_connect).
+
+-include_lib("eunit/include/eunit.hrl").
+-include("emqx.hrl").
+-include("emqx_mqtt.hrl").
+
+-define(BRIDGE_NAME, test).
+-define(BRIDGE_REG_NAME, emqx_bridge_test).
+-define(WAIT(PATTERN, TIMEOUT),
+        receive
+            PATTERN ->
+                ok
+        after
+            TIMEOUT ->
+                error(timeout)
+        end).
+
+%% stub callbacks
+-export([start/1, send/2, stop/2]).
+
+start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) ->
+    case is_pid(Pid) of
+        true -> Pid ! {connection_start_attempt, Ref};
+        false -> ok
+    end,
+    Result.
+
+send(SendFun, Batch) when is_function(SendFun, 1) ->
+    SendFun(Batch).
+
+stop(_Ref, _Pid) -> ok.
+
+%% bridge worker should retry connecting remote node indefinitely
+reconnect_test() ->
+    Ref = make_ref(),
+    Config = make_config(Ref, self(), {error, test}),
+    {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config),
+    %% assert name registered
+    ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
+    ?WAIT({connection_start_attempt, Ref}, 1000),
+    %% expect same message again
+    ?WAIT({connection_start_attempt, Ref}, 1000),
+    ok = emqx_bridge:stop(?BRIDGE_REG_NAME),
+    ok.
+
+%% connect first, disconnect, then connect again
+disturbance_test() ->
+    Ref = make_ref(),
+    Config = make_config(Ref, self(), {ok, Ref, connection}),
+    {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config),
+    ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
+    ?WAIT({connection_start_attempt, Ref}, 1000),
+    Pid ! {disconnected, Ref, test},
+    ?WAIT({connection_start_attempt, Ref}, 1000),
+    ok = emqx_bridge:stop(?BRIDGE_REG_NAME).
+
+%% buffer should continue taking in messages when disconnected
+buffer_when_disconnected_test_() ->
+    {timeout, 10000, fun test_buffer_when_disconnected/0}.
+
+test_buffer_when_disconnected() ->
+    Ref = make_ref(),
+    Nums = lists:seq(1, 100),
+    Sender = spawn_link(fun() -> receive {bridge, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end),
+    SenderMref = monitor(process, Sender),
+    Receiver = spawn_link(fun() -> receive {bridge, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end),
+    ReceiverMref = monitor(process, Receiver),
+    SendFun = fun(Batch) ->
+                      BatchRef = make_ref(),
+                      Receiver ! {batch, BatchRef, Batch},
+                      {ok, BatchRef}
+              end,
+    Config0 = make_config(Ref, false, {ok, Ref, SendFun}),
+    Config = Config0#{reconnect_delay_ms => 100},
+    {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config),
+    Sender ! {bridge, Pid},
+    Receiver ! {bridge, Pid},
+    ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
+    Pid ! {disconnected, Ref, test},
+    ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000),
+    ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000),
+    ok = emqx_bridge:stop(?BRIDGE_REG_NAME).
+
+manual_start_stop_test() ->
+    Ref = make_ref(),
+    Config0 = make_config(Ref, self(), {ok, Ref, connection}),
+    Config = Config0#{start_type := manual},
+    {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config),
+    %% call ensure_started again should yeld the same result
+    {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config),
+    ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
+    ?assertEqual({error, standing_by},
+                 emqx_bridge:ensure_forward_present(Pid, "dummy")),
+    emqx_bridge:ensure_stopped(unknown),
+    emqx_bridge:ensure_stopped(Pid),
+    emqx_bridge:ensure_stopped(?BRIDGE_REG_NAME).
+
+%% Feed messages to bridge
+sender_loop(_Pid, [], _) -> exit(normal);
+sender_loop(Pid, [Num | Rest], Interval) ->
+    random_sleep(Interval),
+    Pid ! {dispatch, dummy, make_msg(Num)},
+    sender_loop(Pid, Rest, Interval).
+
+%% Feed acknowledgments to bridge
+receiver_loop(_Pid, [], _) -> ok;
+receiver_loop(Pid, Nums, Interval) ->
+    receive
+        {batch, BatchRef, Batch} ->
+            Rest = match_nums(Batch, Nums),
+            random_sleep(Interval),
+            emqx_bridge:handle_ack(Pid, BatchRef),
+            receiver_loop(Pid, Rest, Interval)
+    end.
+
+random_sleep(MaxInterval) ->
+    case rand:uniform(MaxInterval) - 1 of
+        0 -> ok;
+        T -> timer:sleep(T)
+    end.
+
+match_nums([], Rest) -> Rest;
+match_nums([#message{payload = P} | Rest], Nums) ->
+    I = binary_to_integer(P),
+    case Nums of
+        [I | NumsLeft] -> match_nums(Rest, NumsLeft);
+        [J | _] when J > I -> match_nums(Rest, Nums); %% allow retry
+        _ -> error([{received, I}, {expecting, Nums}])
+    end.
+
+make_config(Ref, TestPid, Result) ->
+    #{test_pid => TestPid,
+      test_ref => Ref,
+      connect_module => ?MODULE,
+      reconnect_delay_ms => 50,
+      connect_result => Result,
+      start_type => auto
+     }.
+
+make_msg(I) ->
+    Payload = integer_to_binary(I),
+    emqx_message:make(<<"test/topic">>, Payload).

test/emqx_broker_SUITE.erl

@@ -29,8 +29,7 @@ all() ->
     [{group, pubsub},
      {group, session},
      {group, metrics},
-     {group, stats},
+     {group, stats}].
-     {group, alarms}].
 
 groups() ->
     [
@@ -41,8 +40,7 @@ groups() ->
                            'pubsub#', 'pubsub+']},
      {session, [sequence], [start_session]},
      {metrics, [sequence], [inc_dec_metric]},
-     {stats, [sequence], [set_get_stat]},
+     {stats, [sequence], [set_get_stat]}
-     {alarms, [sequence], [set_alarms]}
     ].
 
 init_per_suite(Config) ->
@@ -171,12 +169,3 @@ inc_dec_metric(_) ->
 set_get_stat(_) ->
     emqx_stats:setstat('retained/max', 99),
     99 = emqx_stats:getstat('retained/max').
-
-set_alarms(_) ->
-    AlarmTest = #alarm{id = <<"1">>, severity = error, title="alarm title", summary="alarm summary"},
-    emqx_alarm_mgr:set_alarm(AlarmTest),
-    Alarms = emqx_alarm_mgr:get_alarms(),
-    ct:log("Alarms Length: ~p ~n", [length(Alarms)]),
-    ?assertEqual(1, length(Alarms)),
-    emqx_alarm_mgr:clear_alarm(<<"1">>),
-    [] = emqx_alarm_mgr:get_alarms().

test/emqx_connection_SUITE.erl

@@ -23,59 +23,6 @@
 
 -include("emqx_mqtt.hrl").
 
--define(STATS, [{mailbox_len, _},
-                {heap_size, _},
-                {reductions, _},
-                {recv_pkt, _},
-                {recv_msg, _},
-                {send_pkt, _},
-                {send_msg, _},
-                {recv_oct, _},
-                {recv_cnt, _},
-                {send_oct, _},
-                {send_cnt, _},
-                {send_pend, _}]).
-
--define(ATTRS, [{clean_start, _},
-                {client_id, _},
-                {connected_at, _},
-                {is_bridge, _},
-                {is_super, _},
-                {keepalive, _},
-                {mountpoint, _},
-                {peercert, _},
-                {peername, _},
-                {proto_name, _},
-                {proto_ver, _},
-                {sockname, _},
-                {username, _},
-                {zone, _}]).
-
--define(INFO, [{ack_props, _},
-               {active_n, _},
-               {clean_start, _},
-               {client_id, _},
-               {conn_props, _},
-               {conn_state, _},
-               {connected_at, _},
-               {enable_acl, _},
-               {is_bridge, _},
-               {is_super, _},
-               {keepalive, _},
-               {mountpoint, _},
-               {peercert, _},
-               {peername, _},
-               {proto_name, _},
-               {proto_ver, _},
-               {pub_limit, _},
-               {rate_limit, _},
-               {session, _},
-               {sockname, _},
-               {socktype, _},
-               {topic_aliases, _},
-               {username, _},
-               {zone, _}]).
-
 all() ->
     [t_connect_api].
 
@@ -93,9 +40,33 @@ t_connect_api(_Config) ->
                                        {password, <<"pass1">>}]),
     {ok, _} = emqx_client:connect(T1),
     CPid = emqx_cm:lookup_conn_pid(<<"client1">>),
-    ?STATS = emqx_connection:stats(CPid),
+    ConnStats = emqx_connection:stats(CPid),
-    ?ATTRS = emqx_connection:attrs(CPid),
+    ok = t_stats(ConnStats),
-    ?INFO = emqx_connection:info(CPid),
+    ConnAttrs = emqx_connection:attrs(CPid),
+    ok = t_attrs(ConnAttrs),
+    ConnInfo = emqx_connection:info(CPid),
+    ok = t_info(ConnInfo),
     SessionPid = emqx_connection:session(CPid),
     true = is_pid(SessionPid),
     emqx_client:disconnect(T1).
+
+t_info(ConnInfo) ->
+    ?assertEqual(tcp, proplists:get_value(socktype, ConnInfo)),
+    ?assertEqual(running, proplists:get_value(conn_state, ConnInfo)),
+    ?assertEqual(<<"client1">>, proplists:get_value(client_id, ConnInfo)),
+    ?assertEqual(<<"testuser1">>, proplists:get_value(username, ConnInfo)),
+    ?assertEqual(<<"MQTT">>, proplists:get_value(proto_name, ConnInfo)).
+
+t_attrs(AttrsData) ->
+    ?assertEqual(<<"client1">>, proplists:get_value(client_id, AttrsData)),
+    ?assertEqual(emqx_connection, proplists:get_value(conn_mod, AttrsData)),  
+    ?assertEqual(<<"testuser1">>, proplists:get_value(username, AttrsData)).
+
+t_stats(StatsData) ->
+    ?assertEqual(true, proplists:get_value(recv_oct, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(mailbox_len, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(heap_size, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(reductions, StatsData) >=0),
+    ?assertEqual(true, proplists:get_value(recv_pkt, StatsData) =:=1),
+    ?assertEqual(true, proplists:get_value(recv_msg, StatsData) >=0),
+    ?assertEqual(true, proplists:get_value(send_pkt, StatsData) =:=1).

test/emqx_ct_broker_helpers.erl

@@ -54,10 +54,17 @@
                          "ECDH-RSA-AES128-SHA","AES128-SHA"]}]).
 
 run_setup_steps() ->
+    _ = run_setup_steps([]),
+    %% return ok to be backward compatible
+    ok.
+
+run_setup_steps(Config) ->
     NewConfig = generate_config(),
     lists:foreach(fun set_app_env/1, NewConfig),
     set_bridge_env(),
-    application:ensure_all_started(?APP).
+    {ok, _} = application:ensure_all_started(?APP),
+    set_log_level(Config),
+    Config.
 
 run_teardown_steps() ->
     ?APP:shutdown().
@@ -67,6 +74,12 @@ generate_config() ->
     Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]),
     cuttlefish_generator:map(Schema, Conf).
 
+set_log_level(Config) ->
+    case proplists:get_value(log_level, Config) of
+        undefined -> ok;
+        Level -> emqx_logger:set_log_level(Level)
+    end.
+
 get_base_dir(Module) ->
     {file, Here} = code:is_loaded(Module),
     filename:dirname(filename:dirname(Here)).
@@ -156,24 +169,30 @@ flush(Msgs) ->
     end.
 
 bridge_conf() ->
-    [{aws,
+    [ {local_rpc,
-      [{username,"user"},
+        [{connect_module, emqx_bridge_rpc},
-       {address,"127.0.0.1:1883"},
+         {address, node()},
-       {clean_start,true},
+         {forwards, ["bridge-1/#", "bridge-2/#"]}
-       {client_id,"bridge_aws"},
+        ]}
-       {forwards,["topic1/#","topic2/#"]},
+    ].
-       {keepalive,60000},
+    % [{aws,
-       {max_inflight,32},
+    %   [{connect_module, emqx_bridge_mqtt},
-       {mountpoint,"bridge/aws/${node}/"},
+    %   {username,"user"},
-       {password,"passwd"},
+    %    {address,"127.0.0.1:1883"},
-       {proto_ver,mqttv4},
+    %    {clean_start,true},
-       {queue,
+    %    {client_id,"bridge_aws"},
-        #{batch_size => 1000,mem_cache => true,
+    %    {forwards,["topic1/#","topic2/#"]},
-          replayq_dir => "data/emqx_aws_bridge/",
+    %    {keepalive,60000},
-          replayq_seg_bytes => 10485760}},
+    %    {max_inflight,32},
-       {reconnect_interval,30000},
+    %    {mountpoint,"bridge/aws/${node}/"},
-       {retry_interval,20000},
+    %    {password,"passwd"},
-       {ssl,false},
+    %    {proto_ver,mqttv4},
-       {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]},
+    %    {queue,
-       {start_type,manual},
+    %     #{batch_coun t_limit => 1000,
-       {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}].
+    %       replayq_dir => "data/emqx_aws_bridge/",
+    %       replayq_seg_bytes => 10485760}},
+    %    {reconnect_delay_ms,30000},
+    %    {ssl,false},
+    %    {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]},
+    %    {start_type,manual},
+    %    {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}].

test/emqx_ct_helpers.erl

@@ -14,9 +14,55 @@
 
 -module(emqx_ct_helpers).
 
--export([ensure_mnesia_stopped/0]).
+-export([ensure_mnesia_stopped/0, wait_for/4]).
 
 ensure_mnesia_stopped() ->
     ekka_mnesia:ensure_stopped(),
     ekka_mnesia:delete_schema().
 
+%% Help function to wait for Fun to yield 'true'.
+wait_for(Fn, Ln, F, Timeout) ->
+    {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end),
+    wait_for_down(Fn, Ln, Timeout, Pid, Mref, false).
+
+wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) ->
+    receive
+        {'DOWN', Mref, process, Pid, normal} ->
+            ok;
+        {'DOWN', Mref, process, Pid, {unexpected, Result}} ->
+            erlang:error({unexpected, Fn, Ln, Result});
+        {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} ->
+            erlang:raise(C, {Fn, Ln, E}, S)
+    after
+        Timeout ->
+            case Kill of
+                true ->
+                    erlang:demonitor(Mref, [flush]),
+                    erlang:exit(Pid, kill),
+                    erlang:error({Fn, Ln, timeout});
+                false ->
+                    Pid ! stop,
+                    wait_for_down(Fn, Ln, Timeout, Pid, Mref, true)
+            end
+    end.
+
+wait_loop(_F, ok) -> exit(normal);
+wait_loop(F, LastRes) ->
+    receive
+        stop -> erlang:exit(LastRes)
+    after
+        100 ->
+            Res = catch_call(F),
+            wait_loop(F, Res)
+    end.
+
+catch_call(F) ->
+    try
+        case F() of
+            true -> ok;
+            Other -> {unexpected, Other}
+        end
+    catch
+        C : E : S ->
+            {crashed, {C, E, S}}
+    end.

test/emqx_misc_tests.erl

@@ -47,12 +47,14 @@ timer_cancel_flush_test() ->
     end.
 
 shutdown_disabled_test() ->
+    ok = drain(),
     self() ! foo,
     ?assertEqual(continue, conn_proc_mng_policy(0)),
     receive foo -> ok end,
     ?assertEqual(hibernate, conn_proc_mng_policy(0)).
 
 message_queue_too_long_test() ->
+    ok = drain(),
     self() ! foo,
     self() ! bar,
     ?assertEqual({shutdown, message_queue_too_long},
@@ -63,3 +65,18 @@ message_queue_too_long_test() ->
 
 conn_proc_mng_policy(L) ->
     emqx_misc:conn_proc_mng_policy(#{message_queue_len => L}).
+
+%% drain self() msg queue for deterministic test behavior
+drain() ->
+    _ = drain([]), % maybe log
+    ok.
+
+drain(Acc) ->
+    receive
+        Msg ->
+            drain([Msg | Acc])
+    after
+        0 ->
+            lists:reverse(Acc)
+    end.
+

test/emqx_mqtt_packet_SUITE.erl

@@ -1,18 +1,16 @@
-%%%===================================================================
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%% Copyright (c) 2013-2013-2019 EMQ Inc. All rights reserved.
+%%
-%%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
-%%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
-%%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
-%%% You may obtain a copy of the License at
+%%
-%%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
-%%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
-%%%
+%% Unless required by applicable law or agreed to in writing, software
-%%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
-%%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
-%%% See the License for the specific language governing permissions and
+%% limitations under the License.
-%%% limitations under the License.
-%%%===================================================================
 
 -module(emqx_mqtt_packet_SUITE).
 
@@ -90,7 +88,7 @@ case1_protocol_name(_) ->
     {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data),
     Disconnect  = gen_tcp:recv(Sock, 0),
     ?assertEqual({error, closed}, Disconnect).
- 
+
 case2_protocol_ver(_) ->
     {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000),
     Packet = serialize(?CASE2_PROTOCAL_VER),

test/emqx_os_mon_SUITE.erl

@@ -0,0 +1,56 @@
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_os_mon_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-include_lib("common_test/include/ct.hrl").
+
+all() -> [t_api].
+
+init_per_suite(Config) ->
+    application:ensure_all_started(os_mon),
+    Config.
+
+end_per_suite(_Config) ->
+    application:stop(os_mon).
+
+t_api(_) ->
+    gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+    {ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1},
+                                      {cpu_high_watermark, 0.05},
+                                      {cpu_low_watermark, 0.80},
+                                      {mem_check_interval, 60},
+                                      {sysmem_high_watermark, 0.70},
+                                      {procmem_high_watermark, 0.05}]),
+    ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()),
+    ?assertEqual(0.05, emqx_os_mon:get_cpu_high_watermark()),
+    ?assertEqual(0.80, emqx_os_mon:get_cpu_low_watermark()),
+    ?assertEqual(60, emqx_os_mon:get_mem_check_interval()),
+    ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()),
+    ?assertEqual(0.05, emqx_os_mon:get_procmem_high_watermark()),
+    % timer:sleep(2000),
+    % ?assertEqual(true, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())),
+
+    emqx_os_mon:set_cpu_high_watermark(0.8),
+    emqx_os_mon:set_cpu_low_watermark(0.75),
+    ?assertEqual(0.8, emqx_os_mon:get_cpu_high_watermark()),
+    ?assertEqual(0.75, emqx_os_mon:get_cpu_low_watermark()),
+    % timer:sleep(3000),
+    % ?assertEqual(false, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())),
+    ok.

test/emqx_pool_SUITE.erl

@@ -62,7 +62,7 @@ async_submit_mfa(_Config) ->
     emqx_pool:async_submit(fun ?MODULE:test_mfa/0, []).
 
 async_submit_crash(_) ->
-    emqx_pool:async_submit(fun() -> A = 1, A = 0 end).
+    emqx_pool:async_submit(fun() -> error(unexpected_error) end).
 
 t_unexpected(_) ->
     Pid = emqx_pool:worker(),
@@ -73,3 +73,4 @@ t_unexpected(_) ->
 
 test_mfa() ->
     lists:foldl(fun(X, Sum) -> X + Sum end, 0, [1,2,3,4,5]).
+

test/emqx_protocol_SUITE.erl

@@ -1,5 +1,4 @@
-%%--------------------------------------------------------------------
+%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
-%% Copyright (c) 2013-2013-2019 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -12,7 +11,6 @@
 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
-%%--------------------------------------------------------------------
 
 -module(emqx_protocol_SUITE).
 
@@ -33,64 +31,66 @@
                                 username  = <<"emqx">>,
                                 password  = <<"public">>})).
 
--record(pstate, {
+% -record(pstate, {
-          zone,
+%           zone,
-          sendfun,
+%           sendfun,
-          peername,
+%           peername,
-          peercert,
+%           peercert,
-          proto_ver,
+%           proto_ver,
-          proto_name,
+%           proto_name,
-          client_id,
+%           client_id,
-          is_assigned,
+%           is_assigned,
-          conn_pid,
+%           conn_pid,
-          conn_props,
+%           conn_props,
-          ack_props,
+%           ack_props,
-          username,
+%           username,
-          session,
+%           session,
-          clean_start,
+%           clean_start,
-          topic_aliases,
+%           topic_aliases,
-          packet_size,
+%           packet_size,
-          keepalive,
+%           keepalive,
-          mountpoint,
+%           mountpoint,
-          is_super,
+%           is_super,
-          is_bridge,
+%           is_bridge,
-          enable_ban,
+%           enable_ban,
-          enable_acl,
+%           enable_acl,
-          acl_deny_action,
+%           acl_deny_action,
-          recv_stats,
+%           recv_stats,
-          send_stats,
+%           send_stats,
-          connected,
+%           connected,
-          connected_at,
+%           connected_at,
-          ignore_loop,
+%           ignore_loop,
-          topic_alias_maximum
+%           topic_alias_maximum,
-        }).
+%           conn_mod
+%         }).
 
 
--define(TEST_PSTATE(ProtoVer, SendStats),
+% -define(TEST_PSTATE(ProtoVer, SendStats),
-        #pstate{zone                = test,
+%         #pstate{zone                = test,
-                sendfun             = fun(_Packet, _Options) -> ok end,
+%                 sendfun             = fun(_Packet, _Options) -> ok end,
-                peername            = test_peername,
+%                 peername            = test_peername,
-                peercert            = test_peercert,
+%                 peercert            = test_peercert,
-                proto_ver           = ProtoVer,
+%                 proto_ver           = ProtoVer,
-                proto_name          = <<"MQTT">>,
+%                 proto_name          = <<"MQTT">>,
-                client_id           = <<"test_pstate">>,
+%                 client_id           = <<"test_pstate">>,
-                is_assigned         = false,
+%                 is_assigned         = false,
-                conn_pid            = self(),
+%                 conn_pid            = self(),
-                username            = <<"emqx">>,
+%                 username            = <<"emqx">>,
-                is_super            = false,
+%                 is_super            = false,
-                clean_start         = false,
+%                 clean_start         = false,
-                topic_aliases       = #{},
+%                 topic_aliases       = #{},
-                packet_size         = 1000,
+%                 packet_size         = 1000,
-                mountpoint          = <<>>,
+%                 mountpoint          = <<>>,
-                is_bridge           = false,
+%                 is_bridge           = false,
-                enable_ban          = false,
+%                 enable_ban          = false,
-                enable_acl          = true,
+%                 enable_acl          = true,
-                acl_deny_action     = disconnect,
+%                 acl_deny_action     = disconnect,
-                recv_stats          = #{msg => 0, pkt => 0},
+%                 recv_stats          = #{msg => 0, pkt => 0},
-                send_stats          = SendStats,
+%                 send_stats          = SendStats,
-                connected           = false,
+%                 connected           = false,
-                ignore_loop         = false,
+%                 ignore_loop         = false,
-                topic_alias_maximum = #{to_client => 0, from_client => 0}}).
+%                 topic_alias_maximum = #{to_client => 0, from_client => 0},
+%                 conn_mod            = emqx_connection}).
 
 all() ->
     [
@@ -112,8 +112,7 @@ groups() ->
       [connect_v5,
        subscribe_v5]},
      {acl, [sequence],
-      [acl_deny_action_ct,
+      [acl_deny_action_ct]}].
-       acl_deny_action_eunit]}].
 
 init_per_suite(Config) ->
     [start_apps(App, SchemaFile, ConfigFile) ||
@@ -571,13 +570,13 @@ acl_deny_action_ct(_) ->
     emqx_zone:set_env(external, acl_deny_action, ignore),
     ok.
 
-acl_deny_action_eunit(_) ->
+% acl_deny_action_eunit(_) ->
-    PState = ?TEST_PSTATE(?MQTT_PROTO_V5, #{msg => 0, pkt => 0}),
+%     PState = ?TEST_PSTATE(?MQTT_PROTO_V5, #{msg => 0, pkt => 0}),
-    CodeName = emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ?MQTT_PROTO_V5),
+%     CodeName = emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ?MQTT_PROTO_V5),
-    {error, CodeName, NEWPSTATE1} = emqx_protocol:process_packet(?PUBLISH_PACKET(?QOS_1, <<"acl_deny_action">>, 1, <<"payload">>), PState),
+%     {error, CodeName, NEWPSTATE1} = emqx_protocol:process(?PUBLISH_PACKET(?QOS_1, <<"acl_deny_action">>, 1, <<"payload">>), PState),
-    ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE1#pstate.send_stats),
+%     ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE1#pstate.send_stats),
-    {error, CodeName, NEWPSTATE2} = emqx_protocol:process_packet(?PUBLISH_PACKET(?QOS_2, <<"acl_deny_action">>, 2, <<"payload">>), PState),
+%     {error, CodeName, NEWPSTATE2} = emqx_protocol:process(?PUBLISH_PACKET(?QOS_2, <<"acl_deny_action">>, 2, <<"payload">>), PState),
-    ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE2#pstate.send_stats).
+%     ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE2#pstate.send_stats).
 
 will_topic_check(_) ->
     {ok, Client} = emqx_client:start_link([{username, <<"emqx">>},

test/emqx_session_SUITE.erl

@@ -1,4 +1,3 @@
-
 %% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
@@ -45,7 +44,7 @@ ignore_loop(_Config) ->
     application:set_env(emqx, mqtt_ignore_loop_deliver, false).
 
 t_session_all(_) ->
-    emqx_zone:set_env(internal, idle_timeout, 100),
+    emqx_zone:set_env(internal, idle_timeout, 1000),
     ClientId = <<"ClientId">>,
     {ok, ConnPid} = emqx_mock_client:start_link(ClientId),
     {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal),
@@ -56,7 +55,7 @@ t_session_all(_) ->
     [{<<"topic">>, _}] = emqx:subscriptions(SPid),
     emqx_session:publish(SPid, 1, Message1),
     timer:sleep(200),
-    {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid),
+    [{publish, 1, _}] = emqx_mock_client:get_last_message(ConnPid),
     Attrs = emqx_session:attrs(SPid),
     Info = emqx_session:info(SPid),
     Stats = emqx_session:stats(SPid),

test/emqx_shared_sub_SUITE.erl

@@ -29,7 +29,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--define(wait(For, Timeout), wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)).
+-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)).
 
 all() -> [t_random_basic,
           t_random,
@@ -59,7 +59,7 @@ t_random_basic(_) ->
     PacketId = 1,
     emqx_session:publish(SPid, PacketId, Message1),
     ?wait(case emqx_mock_client:get_last_message(ConnPid) of
-              {publish, 1, _} -> true;
+              [{publish, 1, _}] -> true;
               Other -> Other
           end, 1000),
     emqx_session:pubrec(SPid, PacketId, reasoncode),
@@ -105,7 +105,7 @@ t_no_connection_nack(_) ->
         fun(PacketId, ConnPid) ->
                 Payload = MkPayload(PacketId),
                 case emqx_mock_client:get_last_message(ConnPid) of
-                    {publish, _, #message{payload = Payload}} ->
+                    [{publish, _, #message{payload = Payload}}] ->
                         CasePid ! {Ref, PacketId, ConnPid},
                         true;
                     _Other ->
@@ -176,7 +176,7 @@ t_not_so_sticky(_) ->
     ?wait(subscribed(<<"group1">>, <<"foo/bar">>, SPid1), 1000),
     emqx_session:publish(SPid1, 1, Message1),
     ?wait(case emqx_mock_client:get_last_message(ConnPid1) of
-              {publish, _, #message{payload = <<"hello1">>}} -> true;
+              [{publish, _, #message{payload = <<"hello1">>}}] -> true;
               Other -> Other
           end, 1000),
     emqx_mock_client:close_session(ConnPid1),
@@ -185,7 +185,7 @@ t_not_so_sticky(_) ->
     ?wait(subscribed(<<"group1">>, <<"foo/#">>, SPid2), 1000),
     emqx_session:publish(SPid2, 2, Message2),
     ?wait(case emqx_mock_client:get_last_message(ConnPid2) of
-              {publish, _, #message{payload = <<"hello2">>}} -> true;
+              [{publish, _, #message{payload = <<"hello2">>}}] -> true;
               Other -> Other
           end, 1000),
     emqx_mock_client:close_session(ConnPid2),
@@ -240,7 +240,7 @@ test_two_messages(Strategy, WithAck) ->
 last_message(_ExpectedPayload, []) -> <<"not yet?">>;
 last_message(ExpectedPayload, [Pid | Pids]) ->
     case emqx_mock_client:get_last_message(Pid) of
-        {publish, _, #message{payload = ExpectedPayload}} -> {true, Pid};
+        [{publish, _, #message{payload = ExpectedPayload}}] -> {true, Pid};
         _Other -> last_message(ExpectedPayload, Pids)
     end.
 
@@ -259,49 +259,3 @@ ensure_config(Strategy, AckEnabled) ->
 subscribed(Group, Topic, Pid) ->
     lists:member(Pid, emqx_shared_sub:subscribers(Group, Topic)).
 
-wait_for(Fn, Ln, F, Timeout) ->
-    {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end),
-    wait_for_down(Fn, Ln, Timeout, Pid, Mref, false).
-
-wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) ->
-    receive
-        {'DOWN', Mref, process, Pid, normal} ->
-            ok;
-        {'DOWN', Mref, process, Pid, {unexpected, Result}} ->
-            erlang:error({unexpected, Fn, Ln, Result});
-        {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} ->
-            erlang:raise(C, {Fn, Ln, E}, S)
-    after
-        Timeout ->
-            case Kill of
-                true ->
-                    erlang:demonitor(Mref, [flush]),
-                    erlang:exit(Pid, kill),
-                    erlang:error({Fn, Ln, timeout});
-                false ->
-                    Pid ! stop,
-                    wait_for_down(Fn, Ln, Timeout, Pid, Mref, true)
-            end
-    end.
-
-wait_loop(_F, ok) -> exit(normal);
-wait_loop(F, LastRes) ->
-    receive
-        stop -> erlang:exit(LastRes)
-    after
-        100 ->
-            Res = catch_call(F),
-            wait_loop(F, Res)
-    end.
-
-catch_call(F) ->
-    try
-        case F() of
-            true -> ok;
-            Other -> {unexpected, Other}
-        end
-    catch
-        C : E : S ->
-            {crashed, {C, E, S}}
-    end.
-

test/emqx_vm_mon_SUITE.erl

@@ -0,0 +1,50 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2013-2013-2019 EMQ Enterprise, Inc. (http://emqtt.io)
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_vm_mon_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-include_lib("common_test/include/ct.hrl").
+
+all() -> [t_api].
+
+init_per_suite(Config) ->
+    application:ensure_all_started(sasl),
+    Config.
+
+end_per_suite(_Config) ->
+    application:stop(sasl).
+
+t_api(_) ->
+    gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+    {ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
+                                      {process_high_watermark, 0},
+                                      {process_low_watermark, 0.6}]),
+    timer:sleep(2000),
+    ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+    emqx_vm_mon:set_process_high_watermark(0.8),
+    emqx_vm_mon:set_process_low_watermark(0.75),
+    ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()),
+    ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()),
+    timer:sleep(3000),
+    ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+    emqx_vm_mon:set_check_interval(20),
+    ?assertEqual(20, emqx_vm_mon:get_check_interval()),
+    ok.

test/emqx_ws_connection_SUITE.erl

@@ -35,56 +35,6 @@
 
 -define(PUBQOS, 1).
 
--define(INFO, [{socktype, _},
-               {conn_state, _},
-               {peername, _},
-               {sockname, _},
-               {zone, _},
-               {client_id, <<"mqtt_client">>},
-               {username, <<"admin">>},
-               {peername, _},
-               {peercert, _},
-               {proto_ver, _},
-               {proto_name, _},
-               {clean_start, _},
-               {keepalive, _},
-               {mountpoint, _},
-               {is_super, _},
-               {is_bridge, _},
-               {connected_at, _},
-               {conn_props, _},
-               {ack_props, _},
-               {session, _},
-               {topic_aliases, _},
-               {enable_acl, _}]).
-
--define(ATTRS, [{clean_start,true},
-                {client_id, <<"mqtt_client">>},
-                {connected_at, _},
-                {is_bridge, _},
-                {is_super, _},
-                {keepalive, _},
-                {mountpoint, _},
-                {peercert, _},
-                {peername, _},
-                {proto_name, _},
-                {proto_ver, _},
-                {sockname, _},
-                {username, <<"admin">>},
-                {zone, _}]).
-
--define(STATS, [{recv_oct, _},
-                {recv_cnt, _},
-                {send_oct, _},
-                {send_cnt, _},
-                {mailbox_len, _},
-                {heap_size, _},
-                {reductions, _},
-                {recv_pkt, _},
-                {recv_msg, _},
-                {send_pkt, _},
-                {send_msg, _}]).
-
 all() ->
     [t_ws_connect_api].
 
@@ -103,9 +53,12 @@ t_ws_connect_api(_Config) ->
     {binary, CONACK} = rfc6455_client:recv(WS),
     {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(CONACK),
     Pid = emqx_cm:lookup_conn_pid(<<"mqtt_client">>),
-    ?INFO = emqx_ws_connection:info(Pid),
+    ConnInfo = emqx_ws_connection:info(Pid),
-    ?ATTRS = emqx_ws_connection:attrs(Pid),
+    ok = t_info(ConnInfo),
-    ?STATS = emqx_ws_connection:stats(Pid),
+    ConnAttrs = emqx_ws_connection:attrs(Pid),
+    ok = t_attrs(ConnAttrs),
+    ConnStats = emqx_ws_connection:stats(Pid),
+    ok = t_stats(ConnStats),
     SessionPid = emqx_ws_connection:session(Pid),
     true = is_pid(SessionPid),
     ok = emqx_ws_connection:kick(Pid),
@@ -118,3 +71,24 @@ raw_send_serialize(Packet) ->
 raw_recv_pase(P) ->
     emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE,
                                  version         => ?MQTT_PROTO_V4} }).
+
+t_info(InfoData) ->
+    ?assertEqual(websocket, proplists:get_value(socktype, InfoData)),
+    ?assertEqual(running, proplists:get_value(conn_state, InfoData)),
+    ?assertEqual(<<"mqtt_client">>, proplists:get_value(client_id, InfoData)),
+    ?assertEqual(<<"admin">>, proplists:get_value(username, InfoData)),
+    ?assertEqual(<<"MQTT">>, proplists:get_value(proto_name, InfoData)).
+
+t_attrs(AttrsData) ->
+    ?assertEqual(<<"mqtt_client">>, proplists:get_value(client_id, AttrsData)),
+    ?assertEqual(emqx_ws_connection, proplists:get_value(conn_mod, AttrsData)),
+    ?assertEqual(<<"admin">>, proplists:get_value(username, AttrsData)).
+
+t_stats(StatsData) ->
+    ?assertEqual(true, proplists:get_value(recv_oct, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(mailbox_len, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(heap_size, StatsData) >= 0),
+    ?assertEqual(true, proplists:get_value(reductions, StatsData) >=0),
+    ?assertEqual(true, proplists:get_value(recv_pkt, StatsData) =:=1),
+    ?assertEqual(true, proplists:get_value(recv_msg, StatsData) >=0),
+    ?assertEqual(true, proplists:get_value(send_pkt, StatsData) =:=1).