Add MQTT 5.0 supports for connection, protocol and session modules

etc/emqx.conf

@@ -702,10 +702,15 @@ listener.tcp.external.acceptors = 16
 ## Value: Number
 listener.tcp.external.max_clients = 102400
 
+## Maximum connection per second.
+##
+## Value: Number
+listener.tcp.external.max_conn_rate = 1000
+
 ## Zone of the external MQTT/TCP listener belonged to.
 ##
 ## Value: String
-listener.tcp.external.zone = devicebound
+listener.tcp.external.zone = external
 
 ## Mountpoint of the MQTT/TCP Listener. All the topics of this
 ## listener will be prefixed with the mount point if this option
@@ -831,10 +836,10 @@ listener.tcp.internal.acceptors = 4
 ## Value: Number
 listener.tcp.internal.max_clients = 102400
 
-## TODO: Zone of the internal MQTT/TCP listener belonged to.
+## Zone of the internal MQTT/TCP listener belonged to.
 ##
 ## Value: String
-## listener.tcp.internal.zone = internal
+listener.tcp.internal.zone = internal
 
 ## Mountpoint of the MQTT/TCP Listener.
 ##
@@ -932,10 +937,15 @@ listener.ssl.external.acceptors = 16
 ## Value: Number
 listener.ssl.external.max_clients = 102400
 
-## TODO: Zone of the external MQTT/SSL listener belonged to.
+## Maximum MQTT/SSL connections per second.
+##
+## Value: Number
+listener.ssl.external.max_conn_rate = 1000
+
+## Zone of the external MQTT/SSL listener belonged to.
 ##
 ## Value: String
-## listener.ssl.external.zone = external
+listener.ssl.external.zone = external
 
 ## Mountpoint of the MQTT/SSL Listener.
 ##
@@ -1166,10 +1176,15 @@ listener.ws.external.acceptors = 4
 ## Value: Number
 listener.ws.external.max_clients = 102400
 
-## TODO: Zone of the external MQTT/WebSocket listener belonged to.
+## Maximum MQTT/WebSocket connections per second.
+##
+## Value: Number
+listener.ws.external.max_conn_rate = 1000
+
+## Zone of the external MQTT/WebSocket listener belonged to.
 ##
 ## Value: String
-## listener.ws.external.zone = external
+listener.ws.external.zone = external
 
 ## Mountpoint of the MQTT/WebSocket Listener.
 ##
@@ -1294,10 +1309,15 @@ listener.wss.external.acceptors = 4
 ## Value: Number
 listener.wss.external.max_clients = 64
 
-## TODO: Zone of the external MQTT/WebSocket/SSL listener belonged to.
+## Maximum MQTT/WebSocket/SSL connections per second.
+##
+## Value: Number
+listener.wss.external.max_conn_rate = 1000
+
+## Zone of the external MQTT/WebSocket/SSL listener belonged to.
 ##
 ## Value: String
-## listener.wss.external.zone = external
+listener.wss.external.zone = external
 
 ## Mountpoint of the MQTT/WebSocket/SSL Listener.
 ##

etc/zone.conf

@@ -0,0 +1,126 @@
+
+## Limits and Capabilities
+
+##--------------------------------------------------------------------
+## Connection
+
+zone.${name}.idle_timeout = 30s
+
+zone.${name}.rate_limit = 10,100
+
+## 10 messages per second, with a bucket 100 messages.
+zone.${name}.publish_limit = 10,100
+
+## Enable stats
+zone.${name}.enable_stats = on
+
+## zone.${name}.shutdown_policy = ???
+
+##--------------------------------------------------------------------
+## Protocol
+
+## Capabilities:
+
+## Maximum length of MQTT clientId allowed.
+##
+## Value: Number [23-65535]
+zone.${name}.max_clientid_len = 1024
+
+## Maximum MQTT packet size allowed.
+##
+## Value: Bytes
+##
+## Default: 64K
+zone.${name}.max_packet_size = 64K
+zone.${name}.max_topic_alias = 0
+zone.${name}.max_qos_allowed = 2
+zone.${name}.retain_available = on
+zone.${name}.wildcard_subscription = on
+zone.${name}.shared_subscription = off
+
+## The backoff for MQTT keepalive timeout.
+## EMQ will kick a MQTT connection out until 'Keepalive * backoff * 2' timeout.
+##
+## Value: Float > 0.5
+zone.${name}.keepalive_backoff = 0.75
+
+##--------------------------------------------------------------------
+## Authentication
+
+zone.${name}.allow_anonymous = true
+
+##--------------------------------------------------------------------
+## Session
+
+zone.${name}.max_subscriptions = 0
+zone.${name}.upgrade_qos = off
+
+## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked.
+##
+## Value: Number
+zone.${name}.max_inflight = 32
+
+## Retry interval for QoS1/2 message delivering.
+##
+## Value: Duration
+zone.${name}.retry_interval = 20s
+
+## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL, 0 means no limit.
+##
+## Value: Number
+zone.${name}.max_awaiting_rel = 100
+
+## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout.
+##
+## Value: Duration
+zone.${name}.await_rel_timeout = 30s
+
+## Whether to ignore loop delivery of messages.
+##
+## Value: true | false
+##
+## Default: false
+zone.${name}.ignore_loop_deliver = false
+
+## Max session expiration time.
+##
+## Value: Duration
+## -d: day
+## -h: hour
+## -m: minute
+## -s: second
+##
+## Default: 2h, 2 hours
+zone.${name}.session_expiry_interval = 2h
+
+##--------------------------------------------------------------------
+## Queue
+
+## Message queue type.
+##
+## Value: simple | priority
+zone.${name}.mqueue_type = simple
+
+## Topic priority. Default is 0.
+##
+## Value: Number [0-255]
+##
+## zone.${name}.mqueue_priority = topic/1=10,topic/2=8
+
+## Maximum queue length. Enqueued messages when persistent client disconnected,
+## or inflight window is full. 0 means no limit.
+##
+## Value: Number >= 0
+zone.${name}.max_mqueue_len = 100
+
+## Whether to enqueue Qos0 messages.
+##
+## Value: false | true
+zone.${name}.mqueue_store_qos0 = true
+
+##--------------------------------------------------------------------
+## General
+
+zone.${name}.enable_stats = on
+
+

include/emqx.hrl

@@ -24,42 +24,51 @@
 
 -define(ERTS_MINIMUM_REQUIRED, "10.0").
 
+%%--------------------------------------------------------------------
+%% PubSub
+%%--------------------------------------------------------------------
+
+-type(pubsub() :: publish | subscribe).
+
+-define(PS(I), (I =:= publish orelse I =:= subscribe)).
+
 %%--------------------------------------------------------------------
 %% Topics' prefix: $SYS | $queue | $share
 %%--------------------------------------------------------------------
 
-%% System Topic
+%% System topic
 -define(SYSTOP, <<"$SYS/">>).
 
-%% Queue Topic
+%% Queue topic
 -define(QUEUE,  <<"$queue/">>).
 
-%% Shared Topic
+%% Shared topic
 -define(SHARE,  <<"$share/">>).
 
 %%--------------------------------------------------------------------
 %% Topic, subscription and subscriber
 %%--------------------------------------------------------------------
 
--type(qos() :: integer()).
-
 -type(topic() :: binary()).
 
--type(suboption() :: {qos, qos()}
+-type(subid() :: binary() | atom()).
-                   | {share, '$queue'}
-                   | {share, binary()}
-                   | {atom(), term()}).
 
--record(subscription, {subid   :: binary() | atom(),
+-type(subopts() :: #{qos => integer(), share => '$queue' | binary(), atom() => term()}).
+
+-record(subscription, {
           topic   :: topic(),
-                       subopts :: list(suboption())}).
+          subid   :: subid(),
+          subopts :: subopts()
+        }).
 
 -type(subscription() :: #subscription{}).
 
--type(subscriber() :: binary() | pid() | {binary(), pid()}).
+-type(subscriber() :: {pid(), subid()}).
+
+-type(topic_table() :: [{topic(), subopts()}]).
 
 %%--------------------------------------------------------------------
-%% Client and session
+%% Client and Session
 %%--------------------------------------------------------------------
 
 -type(protocol() :: mqtt | 'mqtt-sn' | coap | stomp | none | atom()).
@@ -70,18 +79,19 @@
 
 -type(username() :: binary() | atom()).
 
--type(mountpoint() :: binary()).
+-type(zone() :: atom()).
 
--type(zone() :: undefined | atom()).
+-record(client, {
-
+          id          :: client_id(),
--record(client, {id           :: client_id(),
           pid         :: pid(),
           zone        :: zone(),
-                 peername     :: peername(),
-                 username     :: username(),
           protocol    :: protocol(),
-                 attributes   :: #{atom() => term()},
+          peername    :: peername(),
-                 connected_at :: erlang:timestamp()}).
+          peercert    :: nossl | binary(),
+          username    :: username(),
+          clean_start :: boolean(),
+          attributes  :: map()
+         }).
 
 -type(client() :: #client{}).
 
@@ -90,61 +100,51 @@
 -type(session() :: #session{}).
 
 %%--------------------------------------------------------------------
-%% Message and delivery
+%% Payload, Message and Delivery
 %%--------------------------------------------------------------------
 
--type(message_id() :: binary() | undefined).
+-type(qos() :: integer()).
 
--type(message_flag() :: sys | qos | dup | retain | atom()).
+-type(payload() :: binary() | iodata()).
 
--type(message_flags() :: #{message_flag() => boolean() | integer()}).
+-type(message_flag() :: dup | sys | retain | atom()).
-
--type(message_headers() :: #{protocol  => protocol(),
-                             packet_id => pos_integer(),
-                             priority  => non_neg_integer(),
-                             ttl       => pos_integer(),
-                             atom()    => term()}).
-
--type(payload() :: binary()).
 
 %% See 'Application Message' in MQTT Version 5.0
--record(message,
+-record(message, {
-        { id         :: message_id(),      %% Message guid
+          %% Global unique message ID
-          qos        :: qos(),             %% Message qos
+          id :: binary() | pos_integer(),
-          from       :: atom() | client(), %% Message from
+          %% Message QoS
-          sender     :: pid(),             %% The pid of the sender/publisher
+          qos = 0 :: qos(),
-          flags      :: message_flags(),   %% Message flags
+          %% Message from
-          headers    :: message_headers(), %% Message headers
+          from :: atom() | client_id(),
-          topic      :: topic(),           %% Message topic
+          %% Message flags
-          properties :: map(),             %% Message user properties
+          flags :: #{message_flag() => boolean()},
-          payload    :: payload(),         %% Message payload
+          %% Message headers, or MQTT 5.0 Properties
-          timestamp  :: erlang:timestamp() %% Timestamp
+          headers = #{} :: map(),
+          %% Topic that the message is published to
+          topic :: topic(),
+          %% Message Payload
+          payload :: binary(),
+          %% Timestamp
+          timestamp :: erlang:timestamp()
         }).
 
 -type(message() :: #message{}).
 
--record(delivery,
+-record(delivery, {
-        { node    :: node(),    %% The node that created the delivery
+          sender  :: pid(),     %% Sender of the delivery
           message :: message(), %% The message delivered
-          flows   :: list()     %% The message flow path
+          flows   :: list()     %% The dispatch path of message
         }).
 
 -type(delivery() :: #delivery{}).
 
-%%--------------------------------------------------------------------
-%% PubSub
-%%--------------------------------------------------------------------
-
--type(pubsub() :: publish | subscribe).
-
--define(PS(I), (I =:= publish orelse I =:= subscribe)).
-
 %%--------------------------------------------------------------------
 %% Route
 %%--------------------------------------------------------------------
 
--record(route,
+-record(route, {
-        { topic :: topic(),
+          topic :: topic(),
           dest :: node() | {binary(), node()}
          }).
 
@@ -156,20 +156,20 @@
 
 -type(trie_node_id() :: binary() | atom()).
 
--record(trie_node,
+-record(trie_node, {
-        { node_id        :: trie_node_id(),
+          node_id        :: trie_node_id(),
           edge_count = 0 :: non_neg_integer(),
           topic          :: topic() | undefined,
           flags          :: list(atom())
         }).
 
--record(trie_edge,
+-record(trie_edge, {
-        { node_id :: trie_node_id(),
+          node_id :: trie_node_id(),
           word    :: binary() | atom()
         }).
 
--record(trie,
+-record(trie, {
-        { edge    :: #trie_edge{},
+          edge    :: #trie_edge{},
           node_id :: trie_node_id()
         }).
 
@@ -177,11 +177,11 @@
 %% Alarm
 %%--------------------------------------------------------------------
 
--record(alarm,
+-record(alarm, {
-        { id        :: binary(),
+          id        :: binary(),
           severity  :: notice | warning | error | critical,
-          title     :: iolist() | binary(),
+          title     :: iolist(),
-          summary   :: iolist() | binary(),
+          summary   :: iolist(),
           timestamp :: erlang:timestamp()
         }).
 
@@ -191,8 +191,8 @@
 %% Plugin
 %%--------------------------------------------------------------------
 
--record(plugin,
+-record(plugin, {
-        { name    :: atom(),
+          name    :: atom(),
           version :: string(),
           dir     :: string(),
           descr   :: string(),
@@ -207,8 +207,8 @@
 %% Command
 %%--------------------------------------------------------------------
 
--record(command,
+-record(command, {
-        { name      :: atom(),
+          name      :: atom(),
           action    :: atom(),
           args = [] :: list(),
           opts = [] :: list(),

include/emqx_mqtt.hrl

@@ -83,13 +83,12 @@
 %%--------------------------------------------------------------------
 %% MQTT Client
 %%--------------------------------------------------------------------
-
+-record(mqtt_client, {
--record(mqtt_client,
+          client_id     :: binary() | undefined,
-        { client_id     :: binary() | undefined,
           client_pid    :: pid(),
           username      :: binary() | undefined,
           peername      :: {inet:ip_address(), inet:port_number()},
-          clean_sess    :: boolean(),
+          clean_start   :: boolean(),
           proto_ver     :: mqtt_version(),
           keepalive = 0 :: non_neg_integer(),
           will_topic    :: undefined | binary(),
@@ -207,8 +206,8 @@
 %% MQTT Packet Fixed Header
 %%--------------------------------------------------------------------
 
--record(mqtt_packet_header,
+-record(mqtt_packet_header, {
-        { type   = ?RESERVED :: mqtt_packet_type(),
+          type   = ?RESERVED :: mqtt_packet_type(),
           dup    = false     :: boolean(),
           qos    = ?QOS_0    :: mqtt_qos(),
           retain = false     :: boolean()
@@ -235,8 +234,8 @@
 
 -type(mqtt_subopts() :: #mqtt_subopts{}).
 
--record(mqtt_packet_connect,
+-record(mqtt_packet_connect, {
-        { proto_name   = <<"MQTT">>     :: binary(),
+          proto_name   = <<"MQTT">>     :: binary(),
           proto_ver    = ?MQTT_PROTO_V4 :: mqtt_version(),
           is_bridge    = false          :: boolean(),
           clean_start  = true           :: boolean(),
@@ -253,55 +252,55 @@
           password     = undefined      :: undefined | binary()
         }).
 
--record(mqtt_packet_connack,
+-record(mqtt_packet_connack, {
-        { ack_flags   :: 0 | 1,
+          ack_flags   :: 0 | 1,
           reason_code :: mqtt_reason_code(),
           properties  :: mqtt_properties()
         }).
 
--record(mqtt_packet_publish,
+-record(mqtt_packet_publish, {
-        { topic_name :: mqtt_topic(),
+          topic_name :: mqtt_topic(),
           packet_id  :: mqtt_packet_id(),
           properties :: mqtt_properties()
         }).
 
--record(mqtt_packet_puback,
+-record(mqtt_packet_puback, {
-        { packet_id   :: mqtt_packet_id(),
+          packet_id   :: mqtt_packet_id(),
           reason_code :: mqtt_reason_code(),
           properties  :: mqtt_properties()
         }).
 
--record(mqtt_packet_subscribe,
+-record(mqtt_packet_subscribe, {
-        { packet_id     :: mqtt_packet_id(),
+          packet_id     :: mqtt_packet_id(),
           properties    :: mqtt_properties(),
           topic_filters :: [{mqtt_topic(), mqtt_subopts()}]
         }).
 
--record(mqtt_packet_suback,
+-record(mqtt_packet_suback, {
-        { packet_id    :: mqtt_packet_id(),
+          packet_id    :: mqtt_packet_id(),
           properties   :: mqtt_properties(),
           reason_codes :: list(mqtt_reason_code())
         }).
 
--record(mqtt_packet_unsubscribe,
+-record(mqtt_packet_unsubscribe, {
-        { packet_id     :: mqtt_packet_id(),
+          packet_id     :: mqtt_packet_id(),
           properties    :: mqtt_properties(),
           topic_filters :: [mqtt_topic()]
         }).
 
--record(mqtt_packet_unsuback,
+-record(mqtt_packet_unsuback, {
-        { packet_id    :: mqtt_packet_id(),
+          packet_id    :: mqtt_packet_id(),
           properties   :: mqtt_properties(),
           reason_codes :: list(mqtt_reason_code())
         }).
 
--record(mqtt_packet_disconnect,
+-record(mqtt_packet_disconnect, {
-        { reason_code :: mqtt_reason_code(),
+          reason_code :: mqtt_reason_code(),
           properties  :: mqtt_properties()
         }).
 
--record(mqtt_packet_auth,
+-record(mqtt_packet_auth, {
-        { reason_code :: mqtt_reason_code(),
+          reason_code :: mqtt_reason_code(),
           properties  :: mqtt_properties()
         }).
 
@@ -309,8 +308,8 @@
 %% MQTT Control Packet
 %%--------------------------------------------------------------------
 
--record(mqtt_packet,
+-record(mqtt_packet, {
-        { header   :: #mqtt_packet_header{},
+          header   :: #mqtt_packet_header{},
           variable :: #mqtt_packet_connect{}
                     | #mqtt_packet_connack{}
                     | #mqtt_packet_publish{}
@@ -364,9 +363,12 @@
                  variable = #mqtt_packet_auth{reason_code = ReasonCode,
                                               properties = Properties}}).
 
--define(PUBLISH_PACKET(Qos, PacketId),
+-define(PUBLISH_PACKET(QoS),
+    #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, qos = QoS}}).
+
+-define(PUBLISH_PACKET(QoS, PacketId),
     #mqtt_packet{header   = #mqtt_packet_header{type = ?PUBLISH,
-                                                qos = Qos},
+                                                qos = QoS},
                  variable = #mqtt_packet_publish{packet_id = PacketId}}).
 
 -define(PUBLISH_PACKET(QoS, Topic, PacketId, Payload),
@@ -464,6 +466,11 @@
     #mqtt_packet{header   = #mqtt_packet_header{type = ?UNSUBACK},
                  variable = #mqtt_packet_unsuback{packet_id = PacketId}}).
 
+-define(UNSUBACK_PACKET(PacketId, ReasonCodes),
+    #mqtt_packet{header   = #mqtt_packet_header{type = ?UNSUBACK},
+                 variable = #mqtt_packet_unsuback{packet_id    = PacketId,
+                                                  reason_codes = ReasonCodes}}).
+
 -define(UNSUBACK_PACKET(PacketId, Properties, ReasonCodes),
     #mqtt_packet{header   = #mqtt_packet_header{type = ?UNSUBACK},
                  variable = #mqtt_packet_unsuback{packet_id    = PacketId,
@@ -486,43 +493,3 @@
 -define(PACKET(Type),
     #mqtt_packet{header = #mqtt_packet_header{type = Type}}).
 
-%%--------------------------------------------------------------------
-%% MQTT Message
-%%--------------------------------------------------------------------
-
--type(mqtt_msg_id() :: binary() | undefined).
-
--type(mqtt_msg_from() :: atom() | {binary(), undefined | binary()}).
-
--record(mqtt_message,
-        { %% Global unique message ID
-          id              :: mqtt_msg_id(),
-          %% PacketId
-          packet_id       :: mqtt_packet_id(),
-          %% ClientId and Username
-          from            :: mqtt_msg_from(),
-          %% Topic that the message is published to
-          topic           :: binary(),
-          %% Message QoS
-          qos     = ?QOS0 :: mqtt_qos(),
-          %% Message Flags
-          flags   = []    :: [retain | dup | sys],
-          %% Retain flag
-          retain  = false :: boolean(),
-          %% Dup flag
-          dup     = false :: boolean(),
-          %% $SYS flag
-          sys     = false :: boolean(),
-          %% Properties
-          properties = [] :: list(),
-          %% Payload
-          payload         :: binary(),
-          %% Timestamp
-          timestamp       :: erlang:timestamp()
-        }).
-
--type(mqtt_message() :: #mqtt_message{}).
-
--define(WILL_MSG(Qos, Retain, Topic, Props, Payload),
-        #mqtt_message{qos = Qos, retain = Retain, topic = Topic, properties = Props, payload = Payload}).
-

priv/emqx.schema

@@ -933,6 +933,10 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "listener.tcp.$name.max_conn_rate", "emqx.listeners", [
+  {datatype, integer}
+]}.
+
 {mapping, "listener.tcp.$name.zone", "emqx.listeners", [
   {datatype, string}
 ]}.
@@ -1024,6 +1028,10 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "listener.ssl.$name.max_conn_rate", "emqx.listeners", [
+  {datatype, integer}
+]}.
+
 {mapping, "listener.ssl.$name.zone", "emqx.listeners", [
   {datatype, string}
 ]}.
@@ -1165,8 +1173,8 @@ end}.
   {datatype, integer}
 ]}.
 
-{mapping, "listener.ws.$name.rate_limit", "emqx.listeners", [
+{mapping, "listener.ws.$name.max_conn_rate", "emqx.listeners", [
-  {datatype, string}
+  {datatype, integer}
 ]}.
 
 {mapping, "listener.ws.$name.zone", "emqx.listeners", [
@@ -1261,6 +1269,10 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "listener.wss.$name.max_conn_rate", "emqx.listeners", [
+  {datatype, integer}
+]}.
+
 {mapping, "listener.wss.$name.zone", "emqx.listeners", [
   {datatype, string}
 ]}.
@@ -1404,7 +1416,7 @@ end}.
     AccOpts = fun(Prefix) ->
                   case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of
                       [] -> [];
-                      Rules -> [{access, [Access(Rule) || {_, Rule} <- Rules]}]
+                      Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}]
                   end
               end,
 
@@ -1413,9 +1425,10 @@ end}.
     LisOpts = fun(Prefix) ->
                   Filter([{acceptors,   cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)},
                           {max_clients, cuttlefish:conf_get(Prefix ++ ".max_clients", Conf)},
+                          {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)},
                           {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)},
                           {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))},
-                          {rate_limit, cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined)},
+                          %%{rate_limit, cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined)},
                           {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)},
                           {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)},
                           {mountpoint, MountPoint(cuttlefish:conf_get(Prefix ++ ".mountpoint", Conf, undefined))},

src/emqx.erl

@@ -74,7 +74,7 @@ subscribe(Topic) ->
 subscribe(Topic, Subscriber) ->
     emqx_broker:subscribe(iolist_to_binary(Topic), list_to_subid(Subscriber)).
 
--spec(subscribe(topic() | string(), subscriber() | string(), [suboption()]) -> ok | {error, term()}).
+-spec(subscribe(topic() | string(), subscriber() | string(), subopts()) -> ok | {error, term()}).
 subscribe(Topic, Subscriber, Options) ->
     emqx_broker:subscribe(iolist_to_binary(Topic), list_to_subid(Subscriber), Options).
 
@@ -95,11 +95,11 @@ unsubscribe(Topic, Subscriber) ->
 %% PubSub management API
 %%--------------------------------------------------------------------
 
--spec(get_subopts(topic() | string(), subscriber()) -> [suboption()]).
+-spec(get_subopts(topic() | string(), subscriber()) -> subopts()).
 get_subopts(Topic, Subscriber) ->
     emqx_broker:get_subopts(iolist_to_binary(Topic), list_to_subid(Subscriber)).
 
--spec(set_subopts(topic() | string(), subscriber(), [suboption()]) -> ok).
+-spec(set_subopts(topic() | string(), subscriber(), subopts()) -> ok).
 set_subopts(Topic, Subscriber, Options) when is_list(Options) ->
     emqx_broker:set_subopts(iolist_to_binary(Topic), list_to_subid(Subscriber), Options).
 
@@ -110,7 +110,7 @@ topics() -> emqx_router:topics().
 subscribers(Topic) ->
     emqx_broker:subscribers(iolist_to_binary(Topic)).
 
--spec(subscriptions(subscriber() | string()) -> [{topic(), list(suboption())}]).
+-spec(subscriptions(subscriber() | string()) -> [{topic(), subopts()}]).
 subscriptions(Subscriber) ->
     emqx_broker:subscriptions(list_to_subid(Subscriber)).
 

src/emqx_alarm_mgr.erl

@@ -81,7 +81,7 @@ handle_event({set_alarm, Alarm = #alarm{timestamp = undefined}}, State)->
 handle_event({set_alarm, Alarm = #alarm{id = AlarmId}}, State = #state{alarms = Alarms}) ->
     case encode_alarm(Alarm) of
         {ok, Json} ->
-            emqx_broker:safe_publish(alarm_msg(alert, AlarmId, Json));
+            ok = emqx_broker:safe_publish(alarm_msg(alert, AlarmId, Json));
         {error, Reason} ->
             emqx_logger:error("[AlarmMgr] Failed to encode alarm: ~p", [Reason])
     end,
@@ -131,7 +131,9 @@ encode_alarm(#alarm{id = AlarmId, severity = Severity, title = Title,
                            {ts, emqx_time:now_secs(Ts)}]).
 
 alarm_msg(Type, AlarmId, Json) ->
-    emqx_message:make(?ALARM_MGR, #{sys => true}, topic(Type, AlarmId), Json).
+    Msg = emqx_message:make(?ALARM_MGR, topic(Type, AlarmId), Json),
+    emqx_message:set_headers(#{'Content-Type' => <<"application/json">>},
+                             emqx_message:set_flags(#{sys => true}, Msg)).
 
 topic(alert, AlarmId) ->
     emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>);

src/emqx_bridge.erl

@@ -72,8 +72,8 @@ init([Pool, Id, Node, Topic, Options]) ->
 
 parse_opts([], State) ->
     State;
-parse_opts([{qos, Qos} | Opts], State) ->
+parse_opts([{qos, QoS} | Opts], State) ->
-    parse_opts(Opts, State#state{qos = Qos});
+    parse_opts(Opts, State#state{qos = QoS});
 parse_opts([{topic_suffix, Suffix} | Opts], State) ->
     parse_opts(Opts, State#state{topic_suffix= Suffix});
 parse_opts([{topic_prefix, Prefix} | Opts], State) ->

src/emqx_broker.erl

@@ -20,8 +20,10 @@
 
 -export([start_link/2]).
 -export([subscribe/1, subscribe/2, subscribe/3, subscribe/4]).
--export([publish/1, publish/2, safe_publish/1]).
+-export([multi_subscribe/1, multi_subscribe/2, multi_subscribe/3]).
--export([unsubscribe/1, unsubscribe/2]).
+-export([publish/1, safe_publish/1]).
+-export([unsubscribe/1, unsubscribe/2, unsubscribe/3]).
+-export([multi_unsubscribe/1, multi_unsubscribe/2, multi_unsubscribe/3]).
 -export([dispatch/2, dispatch/3]).
 -export([subscriptions/1, subscribers/1, subscribed/2]).
 -export([get_subopts/2, set_subopts/3]).
@@ -31,102 +33,141 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
--record(state, {pool, id, submon}).
+-record(state, {pool, id, submap, submon}).
+-record(subscribe, {topic, subpid, subid, subopts = #{}}).
+-record(unsubscribe, {topic, subpid, subid}).
 
+%% The default request timeout
+-define(TIMEOUT, 60000).
 -define(BROKER, ?MODULE).
--define(TIMEOUT, 120000).
 
 %% ETS tables
 -define(SUBOPTION,    emqx_suboption).
 -define(SUBSCRIBER,   emqx_subscriber).
 -define(SUBSCRIPTION, emqx_subscription).
 
+-define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))).
+
 -spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}).
 start_link(Pool, Id) ->
-    gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)},
+    gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE,
-                          ?MODULE, [Pool, Id], [{hibernate_after, 2000}]).
+                          [Pool, Id], [{hibernate_after, 2000}]).
 
 %%------------------------------------------------------------------------------
-%% Subscribe/Unsubscribe
+%% Subscribe
 %%------------------------------------------------------------------------------
 
--spec(subscribe(topic()) -> ok | {error, term()}).
+-spec(subscribe(topic()) -> ok).
 subscribe(Topic) when is_binary(Topic) ->
     subscribe(Topic, self()).
 
--spec(subscribe(topic(), subscriber()) -> ok | {error, term()}).
+-spec(subscribe(topic(), pid() | subid()) -> ok).
-subscribe(Topic, Subscriber) when is_binary(Topic) ->
+subscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
-    subscribe(Topic, Subscriber, []).
+    subscribe(Topic, SubPid, undefined);
+subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
+    subscribe(Topic, self(), SubId).
 
--spec(subscribe(topic(), subscriber(), [suboption()]) -> ok | {error, term()}).
+-spec(subscribe(topic(), pid() | subid(), subid() | subopts()) -> ok).
-subscribe(Topic, Subscriber, Options) when is_binary(Topic) ->
+subscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
-    subscribe(Topic, Subscriber, Options, ?TIMEOUT).
+    subscribe(Topic, SubPid, SubId, []);
+subscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
+    subscribe(Topic, SubPid, SubId, []);
+subscribe(Topic, SubPid, SubOpts) when is_binary(Topic), is_pid(SubPid), is_map(SubOpts) ->
+    subscribe(Topic, SubPid, undefined, SubOpts);
+subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts) ->
+    subscribe(Topic, self(), SubId, SubOpts).
 
--spec(subscribe(topic(), subscriber(), [suboption()], timeout())
+-spec(subscribe(topic(), pid(), subid(), subopts()) -> ok).
-      -> ok | {error, term()}).
+subscribe(Topic, SubPid, SubId, SubOpts) when is_binary(Topic), is_pid(SubPid),
-subscribe(Topic, Subscriber, Options, Timeout) ->
+                                              ?is_subid(SubId), is_map(SubOpts) ->
-    {Topic1, Options1} = emqx_topic:parse(Topic, Options),
+    Broker = pick(SubPid),
-    SubReq = {subscribe, Topic1, with_subpid(Subscriber), Options1},
+    SubReq = #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts},
-    async_call(pick(Subscriber), SubReq, Timeout).
+    wait_for_reply(async_call(Broker, SubReq), ?TIMEOUT).
 
--spec(unsubscribe(topic()) -> ok | {error, term()}).
+-spec(multi_subscribe(topic_table()) -> ok).
+multi_subscribe(TopicTable) when is_list(TopicTable) ->
+    multi_subscribe(TopicTable, self()).
+
+-spec(multi_subscribe(topic_table(), pid() | subid()) -> ok).
+multi_subscribe(TopicTable, SubPid) when is_pid(SubPid) ->
+    multi_subscribe(TopicTable, SubPid, undefined);
+multi_subscribe(TopicTable, SubId) when ?is_subid(SubId) ->
+    multi_subscribe(TopicTable, self(), SubId).
+
+-spec(multi_subscribe(topic_table(), pid(), subid()) -> ok).
+multi_subscribe(TopicTable, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) ->
+    Broker = pick(SubPid),
+    SubReq = fun(Topic, SubOpts) ->
+                 #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}
+             end,
+    wait_for_replies([async_call(Broker, SubReq(Topic, SubOpts))
+                      || {Topic, SubOpts} <- TopicTable], ?TIMEOUT).
+
+%%------------------------------------------------------------------------------
+%% Unsubscribe
+%%------------------------------------------------------------------------------
+
+-spec(unsubscribe(topic()) -> ok).
 unsubscribe(Topic) when is_binary(Topic) ->
     unsubscribe(Topic, self()).
 
--spec(unsubscribe(topic(), subscriber()) -> ok | {error, term()}).
+-spec(unsubscribe(topic(), pid() | subid()) -> ok).
-unsubscribe(Topic, Subscriber) when is_binary(Topic) ->
+unsubscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
-    unsubscribe(Topic, Subscriber, ?TIMEOUT).
+    unsubscribe(Topic, SubPid, undefined);
+unsubscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
+    unsubscribe(Topic, self(), SubId).
 
--spec(unsubscribe(topic(), subscriber(), timeout()) -> ok | {error, term()}).
+-spec(unsubscribe(topic(), pid(), subid()) -> ok).
-unsubscribe(Topic, Subscriber, Timeout) ->
+unsubscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
-    {Topic1, _} = emqx_topic:parse(Topic),
+    Broker = pick(SubPid),
-    UnsubReq = {unsubscribe, Topic1, with_subpid(Subscriber)},
+    UnsubReq = #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId},
-    async_call(pick(Subscriber), UnsubReq, Timeout).
+    wait_for_reply(async_call(Broker, UnsubReq), ?TIMEOUT).
+
+-spec(multi_unsubscribe([topic()]) -> ok).
+multi_unsubscribe(Topics) ->
+    multi_unsubscribe(Topics, self()).
+
+-spec(multi_unsubscribe([topic()], pid() | subid()) -> ok).
+multi_unsubscribe(Topics, SubPid) when is_pid(SubPid) ->
+    multi_unsubscribe(Topics, SubPid, undefined);
+multi_unsubscribe(Topics, SubId) when ?is_subid(SubId) ->
+    multi_unsubscribe(Topics, self(), SubId).
+
+-spec(multi_unsubscribe([topic()], pid(), subid()) -> ok).
+multi_unsubscribe(Topics, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) ->
+    Broker = pick(SubPid),
+    UnsubReq = fun(Topic) ->
+                   #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}
+               end,
+    wait_for_replies([async_call(Broker, UnsubReq(Topic)) || Topic <- Topics], ?TIMEOUT).
 
 %%------------------------------------------------------------------------------
 %% Publish
 %%------------------------------------------------------------------------------
 
--spec(publish(topic(), payload()) -> delivery() | stopped).
+-spec(publish(message()) -> delivery()).
-publish(Topic, Payload) when is_binary(Topic), is_binary(Payload) ->
+publish(Msg) when is_record(Msg, message) ->
-    publish(emqx_message:make(Topic, Payload)).
+    _ = emqx_tracer:trace(publish, Msg),
-
--spec(publish(message()) -> {ok, delivery()} | {error, stopped}).
-publish(Msg = #message{from = From}) ->
-    %% Hook to trace?
-    _ = trace(publish, From, Msg),
     case emqx_hooks:run('message.publish', [], Msg) of
         {ok, Msg1 = #message{topic = Topic}} ->
-            {ok, route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1))};
+            route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1));
         {stop, Msg1} ->
-            emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg1)]),
+            emqx_logger:warning("Stop publishing: ~p", [Msg]), delivery(Msg1)
-            {error, stopped}
     end.
 
-%% called internally
+%% Called internally
-safe_publish(Msg) ->
+safe_publish(Msg) when is_record(Msg, message) ->
     try
         publish(Msg)
     catch
         _:Error:Stacktrace ->
             emqx_logger:error("[Broker] publish error: ~p~n~p~n~p", [Error, Msg, Stacktrace])
+    after
+        ok
     end.
 
-%%------------------------------------------------------------------------------
+delivery(Msg) ->
-%% Trace
+    #delivery{sender = self(), message = Msg, flows = []}.
-%%------------------------------------------------------------------------------
-
-trace(publish, From, _Msg) when is_atom(From) ->
-    %% Dont' trace '$SYS' publish
-    ignore;
-trace(public, #client{id = ClientId, username = Username},
-      #message{topic = Topic, payload = Payload}) ->
-    emqx_logger:info([{client, ClientId}, {topic, Topic}],
-                     "~s/~s PUBLISH to ~s: ~p", [Username, ClientId, Topic, Payload]);
-trace(public, From, #message{topic = Topic, payload = Payload})
-    when is_binary(From); is_list(From) ->
-    emqx_logger:info([{client, From}, {topic, Topic}],
-                     "~s PUBLISH to ~s: ~p", [From, Topic, Payload]).
 
 %%------------------------------------------------------------------------------
 %% Route
@@ -186,12 +227,8 @@ dispatch(Topic, Delivery = #delivery{message = Msg, flows = Flows}) ->
             Delivery#delivery{flows = [{dispatch, Topic, Count}|Flows]}
     end.
 
-dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
+dispatch({SubPid, _SubId}, Topic, Msg) when is_pid(SubPid) ->
     SubPid ! {dispatch, Topic, Msg};
-dispatch({SubId, SubPid}, Topic, Msg) when is_binary(SubId), is_pid(SubPid) ->
-    SubPid ! {dispatch, Topic, Msg};
-dispatch(SubId, Topic, Msg) when is_binary(SubId) ->
-   emqx_sm:dispatch(SubId, Topic, Msg);
 dispatch({share, _Group, _Sub}, _Topic, _Msg) ->
     ignored.
 
@@ -200,12 +237,11 @@ dropped(<<"$SYS/", _/binary>>) ->
 dropped(_Topic) ->
     emqx_metrics:inc('messages/dropped').
 
-delivery(Msg) ->
+-spec(subscribers(topic()) -> [subscriber()]).
-    #delivery{node = node(), message = Msg, flows = []}.
-
 subscribers(Topic) ->
     try ets:lookup_element(?SUBSCRIBER, Topic, 2) catch error:badarg -> [] end.
 
+-spec(subscriptions(subscriber()) -> [{topic(), subopts()}]).
 subscriptions(Subscriber) ->
     lists:map(fun({_, {share, _Group, Topic}}) ->
                   subscription(Topic, Subscriber);
@@ -216,51 +252,49 @@ subscriptions(Subscriber) ->
 subscription(Topic, Subscriber) ->
     {Topic, ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)}.
 
--spec(subscribed(topic(), subscriber()) -> boolean()).
+-spec(subscribed(topic(), pid() | subid() | subscriber()) -> boolean()).
 subscribed(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) ->
-    ets:member(?SUBOPTION, {Topic, SubPid});
+    length(ets:match_object(?SUBOPTION, {{Topic, {SubPid, '_'}}, '_'}, 1)) == 1;
-subscribed(Topic, SubId) when is_binary(Topic), is_binary(SubId) ->
+subscribed(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) ->
-    length(ets:match_object(?SUBOPTION, {{Topic, {SubId, '_'}}, '_'}, 1)) == 1;
+    length(ets:match_object(?SUBOPTION, {{Topic, {'_', SubId}}, '_'}, 1)) == 1;
-subscribed(Topic, {SubId, SubPid}) when is_binary(Topic), is_binary(SubId), is_pid(SubPid) ->
+subscribed(Topic, {SubPid, SubId}) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) ->
-    ets:member(?SUBOPTION, {Topic, {SubId, SubPid}}).
+    ets:member(?SUBOPTION, {Topic, {SubPid, SubId}}).
 
--spec(get_subopts(topic(), subscriber()) -> [suboption()]).
+-spec(get_subopts(topic(), subscriber()) -> subopts()).
 get_subopts(Topic, Subscriber) when is_binary(Topic) ->
     try ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)
     catch error:badarg -> []
     end.
 
--spec(set_subopts(topic(), subscriber(), [suboption()]) -> boolean()).
+-spec(set_subopts(topic(), subscriber(), subopts()) -> boolean()).
-set_subopts(Topic, Subscriber, Opts) when is_binary(Topic), is_list(Opts) ->
+set_subopts(Topic, Subscriber, Opts) when is_binary(Topic), is_map(Opts) ->
     case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
         [{_, OldOpts}] ->
-            Opts1 = lists:usort(lists:umerge(Opts, OldOpts)),
+            ets:insert(?SUBOPTION, {{Topic, Subscriber}, maps:merge(OldOpts, Opts)});
-            ets:insert(?SUBOPTION, {{Topic, Subscriber}, Opts1});
         [] -> false
     end.
 
-with_subpid(SubPid) when is_pid(SubPid) ->
+async_call(Broker, Req) ->
-    SubPid;
-with_subpid(SubId) when is_binary(SubId) ->
-    {SubId, self()};
-with_subpid({SubId, SubPid}) when is_binary(SubId), is_pid(SubPid) ->
-    {SubId, SubPid}.
-
-async_call(Broker, Msg, Timeout) ->
     From = {self(), Tag = make_ref()},
-    ok = gen_server:cast(Broker, {From, Msg}),
+    ok = gen_server:cast(Broker, {From, Req}),
+    Tag.
+
+wait_for_replies(Tags, Timeout) ->
+    lists:foreach(
+      fun(Tag) ->
+          wait_for_reply(Tag, Timeout)
+      end, Tags).
+
+wait_for_reply(Tag, Timeout) ->
     receive
         {Tag, Reply} -> Reply
     after Timeout ->
-        {error, timeout}
+        exit(timeout)
     end.
 
+%% Pick a broker
 pick(SubPid) when is_pid(SubPid) ->
-    gproc_pool:pick_worker(broker, SubPid);
+    gproc_pool:pick_worker(broker, SubPid).
-pick(SubId) when is_binary(SubId) ->
-    gproc_pool:pick_worker(broker, SubId);
-pick({SubId, SubPid}) when is_binary(SubId), is_pid(SubPid) ->
-    pick(SubPid).
 
 -spec(topics() -> [topic()]).
 topics() -> emqx_router:topics().
@@ -271,33 +305,35 @@ topics() -> emqx_router:topics().
 
 init([Pool, Id]) ->
     true = gproc_pool:connect_worker(Pool, {Pool, Id}),
-    {ok, #state{pool = Pool, id = Id, submon = emqx_pmon:new()}}.
+    {ok, #state{pool = Pool, id = Id, submap = #{}, submon = emqx_pmon:new()}}.
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[Broker] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({From, {subscribe, Topic, Subscriber, Options}}, State) ->
+handle_cast({From, #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}}, State) ->
-    case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
+    Subscriber = {SubPid, SubId},
-        [] ->
+    case ets:member(?SUBOPTION, {Topic, Subscriber}) of
-            Group = proplists:get_value(share, Options),
+        false ->
-            true = do_subscribe(Group, Topic, Subscriber, Options),
+            Group = maps:get(share, SubOpts, undefined),
-            emqx_shared_sub:subscribe(Group, Topic, subpid(Subscriber)),
+            true = do_subscribe(Group, Topic, Subscriber, SubOpts),
-            emqx_router:add_route(From, Topic, destination(Options)),
+            emqx_shared_sub:subscribe(Group, Topic, SubPid),
+            emqx_router:add_route(From, Topic, dest(Group)),
             {noreply, monitor_subscriber(Subscriber, State)};
-        [_] ->
+        true ->
             gen_server:reply(From, ok),
             {noreply, State}
     end;
 
-handle_cast({From, {unsubscribe, Topic, Subscriber}}, State) ->
+handle_cast({From, #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}}, State) ->
+    Subscriber = {SubPid, SubId},
     case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
-        [{_, Options}] ->
+        [{_, SubOpts}] ->
-            Group = proplists:get_value(share, Options),
+            Group = maps:get(share, SubOpts, undefined),
             true = do_unsubscribe(Group, Topic, Subscriber),
-            emqx_shared_sub:unsubscribe(Group, Topic, subpid(Subscriber)),
+            emqx_shared_sub:unsubscribe(Group, Topic, SubPid),
             case ets:member(?SUBSCRIBER, Topic) of
-                false -> emqx_router:del_route(From, Topic, destination(Options));
+                false -> emqx_router:del_route(From, Topic, dest(Group));
                 true  -> gen_server:reply(From, ok)
             end;
         [] -> gen_server:reply(From, ok)
@@ -308,37 +344,22 @@ handle_cast(Msg, State) ->
     emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]),
     {noreply, State}.
 
-handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{submon = SubMon}) ->
+handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #state{submap = SubMap}) ->
-    Subscriber = case SubMon:find(SubPid) of
+    case maps:find(SubPid, SubMap) of
-                     undefined -> SubPid;
+        {ok, SubIds} ->
-                     SubId -> {SubId, SubPid}
+            lists:foreach(fun(SubId) -> subscriber_down({SubPid, SubId}) end, SubIds),
-                 end,
-    Topics = lists:map(fun({_, {share, _, Topic}}) ->
-                           Topic;
-                          ({_, Topic}) ->
-                           Topic
-                       end, ets:lookup(?SUBSCRIPTION, Subscriber)),
-    lists:foreach(
-      fun(Topic) ->
-        case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
-            [{_, Options}] ->
-                Group = proplists:get_value(share, Options),
-                true = do_unsubscribe(Group, Topic, Subscriber),
-                case ets:member(?SUBSCRIBER, Topic) of
-                    false -> emqx_router:del_route(Topic, destination(Options));
-                    true  -> ok
-                end;
-            [] -> ok
-        end
-    end, Topics),
             {noreply, demonitor_subscriber(SubPid, State)};
+        error ->
+            emqx_logger:error("unexpected 'DOWN': ~p, reason: ~p", [SubPid, Reason]),
+            {noreply, State}
+    end;
 
 handle_info(Info, State) ->
     emqx_logger:error("[Broker] unexpected info: ~p", [Info]),
     {noreply, State}.
 
 terminate(_Reason, #state{pool = Pool, id = Id}) ->
-    true = gproc_pool:disconnect_worker(Pool, {Pool, Id}).
+    gproc_pool:disconnect_worker(Pool, {Pool, Id}).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
@@ -347,35 +368,44 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-do_subscribe(Group, Topic, Subscriber, Options) ->
+do_subscribe(Group, Topic, Subscriber, SubOpts) ->
     ets:insert(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}),
     ets:insert(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}),
-    ets:insert(?SUBOPTION, {{Topic, Subscriber}, Options}).
+    ets:insert(?SUBOPTION, {{Topic, Subscriber}, SubOpts}).
 
 do_unsubscribe(Group, Topic, Subscriber) ->
     ets:delete_object(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}),
     ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}),
     ets:delete(?SUBOPTION, {Topic, Subscriber}).
 
-monitor_subscriber(SubPid, State = #state{submon = SubMon}) when is_pid(SubPid) ->
+subscriber_down(Subscriber) ->
-    State#state{submon = SubMon:monitor(SubPid)};
+    Topics = lists:map(fun({_, {share, _, Topic}}) ->
+                           Topic;
+                          ({_, Topic}) ->
+                           Topic
+                       end, ets:lookup(?SUBSCRIPTION, Subscriber)),
+    lists:foreach(fun(Topic) ->
+                      case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of
+                          [{_, SubOpts}] ->
+                              Group = maps:get(share, SubOpts, undefined),
+                              true = do_unsubscribe(Group, Topic, Subscriber),
+                              ets:member(?SUBSCRIBER, Topic)
+                                orelse emqx_router:del_route(Topic, dest(Group));
+                          [] -> ok
+                      end
+                  end, Topics).
 
-monitor_subscriber({SubId, SubPid}, State = #state{submon = SubMon}) ->
+monitor_subscriber({SubPid, SubId}, State = #state{submap = SubMap, submon = SubMon}) ->
-    State#state{submon = SubMon:monitor(SubPid, SubId)}.
+    UpFun = fun(SubIds) -> lists:usort([SubId|SubIds]) end,
+    State#state{submap = maps:update_with(SubPid, UpFun, [SubId], SubMap),
+                submon = emqx_pmon:monitor(SubPid, SubMon)}.
 
-demonitor_subscriber(SubPid, State = #state{submon = SubMon}) ->
+demonitor_subscriber(SubPid, State = #state{submap = SubMap, submon = SubMon}) ->
-    State#state{submon = SubMon:demonitor(SubPid)}.
+    State#state{submap = maps:remove(SubPid, SubMap),
+                submon = emqx_pmon:demonitor(SubPid, SubMon)}.
 
-destination(Options) ->
+dest(undefined) -> node();
-    case proplists:get_value(share, Options) of
+dest(Group)     -> {Group, node()}.
-       undefined -> node();
-       Group     -> {Group, node()}
-    end.
-
-subpid(SubPid) when is_pid(SubPid) ->
-    SubPid;
-subpid({_SubId, SubPid}) when is_pid(SubPid) ->
-    SubPid.
 
 shared(undefined, Name) -> Name;
 shared(Group, Name)     -> {share, Group, Name}.

src/emqx_broker_helper.erl

@@ -17,9 +17,7 @@
 -behaviour(gen_server).
 
 -export([start_link/0]).
-
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
-         code_change/3]).
 
 -define(HELPER, ?MODULE).
 
@@ -39,7 +37,7 @@ init([]) ->
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]),
-    {reply, ignore, State}.
+   {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
     emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]),

src/emqx_client.erl

@@ -69,6 +69,11 @@
 
 -export_type([host/0, option/0]).
 
+-record(mqtt_msg, {qos = ?QOS0, retain = false, dup = false,
+                   packet_id, topic, props, payload}).
+
+-type(mqtt_msg() :: #mqtt_msg{}).
+
 -record(state, {name            :: atom(),
                 owner           :: pid(),
                 host            :: host(),
@@ -89,7 +94,7 @@
                 force_ping      :: boolean(),
                 paused          :: boolean(),
                 will_flag       :: boolean(),
-                will_msg        :: mqtt_message(),
+                will_msg        :: mqtt_msg(),
                 properties      :: properties(),
                 pending_calls   :: list(),
                 subscriptions   :: map(),
@@ -140,6 +145,9 @@
 
 -define(PROPERTY(Name, Val), #state{properties = #{Name := Val}}).
 
+-define(WILL_MSG(QoS, Retain, Topic, Props, Payload),
+        #mqtt_msg{qos = QoS, retain = Retain, topic = Topic, props = Props, payload = Payload}).
+
 %%------------------------------------------------------------------------------
 %% API
 %%------------------------------------------------------------------------------
@@ -242,8 +250,7 @@ parse_subopt([{qos, QoS} | Opts], Rec) ->
 
 -spec(publish(client(), topic(), payload()) -> ok | {error, term()}).
 publish(Client, Topic, Payload) when is_binary(Topic) ->
-    publish(Client, #mqtt_message{topic = Topic, qos = ?QOS_0,
+    publish(Client, #mqtt_msg{topic = Topic, qos = ?QOS_0, payload = iolist_to_binary(Payload)}).
-                                  payload = iolist_to_binary(Payload)}).
 
 -spec(publish(client(), topic(), payload(), qos() | [pubopt()])
       -> ok | {ok, packet_id()} | {error, term()}).
@@ -261,15 +268,14 @@ publish(Client, Topic, Properties, Payload, Opts)
     ok = emqx_mqtt_properties:validate(Properties),
     Retain = proplists:get_bool(retain, Opts),
     QoS = ?QOS_I(proplists:get_value(qos, Opts, ?QOS_0)),
-    publish(Client, #mqtt_message{qos        = QoS,
+    publish(Client, #mqtt_msg{qos     = QoS,
                               retain  = Retain,
                               topic   = Topic,
-                                  properties = Properties,
+                              props   = Properties,
                               payload = iolist_to_binary(Payload)}).
 
--spec(publish(client(), mqtt_message())
+-spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}).
-      -> ok | {ok, packet_id()} | {error, term()}).
+publish(Client, Msg) when is_record(Msg, mqtt_msg) ->
-publish(Client, Msg) when is_record(Msg, mqtt_message) ->
     gen_statem:call(Client, {publish, Msg}).
 
 -spec(unsubscribe(client(), topic() | [topic()]) -> subscribe_ret()).
@@ -380,7 +386,7 @@ init([Options]) ->
                                  force_ping      = false,
                                  paused          = false,
                                  will_flag       = false,
-                                 will_msg        = #mqtt_message{},
+                                 will_msg        = #mqtt_msg{},
                                  pending_calls   = [],
                                  subscriptions   = #{},
                                  max_inflight    = infinity,
@@ -488,15 +494,15 @@ init([_Opt | Opts], State) ->
     init(Opts, State).
 
 init_will_msg({topic, Topic}, WillMsg) ->
-    WillMsg#mqtt_message{topic = iolist_to_binary(Topic)};
+    WillMsg#mqtt_msg{topic = iolist_to_binary(Topic)};
-init_will_msg({props, Properties}, WillMsg) ->
+init_will_msg({props, Props}, WillMsg) ->
-    WillMsg#mqtt_message{properties = Properties};
+    WillMsg#mqtt_msg{props = Props};
 init_will_msg({payload, Payload}, WillMsg) ->
-    WillMsg#mqtt_message{payload = iolist_to_binary(Payload)};
+    WillMsg#mqtt_msg{payload = iolist_to_binary(Payload)};
 init_will_msg({retain, Retain}, WillMsg) when is_boolean(Retain) ->
-    WillMsg#mqtt_message{retain = Retain};
+    WillMsg#mqtt_msg{retain = Retain};
 init_will_msg({qos, QoS}, WillMsg) ->
-    WillMsg#mqtt_message{qos = ?QOS_I(QoS)}.
+    WillMsg#mqtt_msg{qos = ?QOS_I(QoS)}.
 
 init_parse_state(State = #state{proto_ver = Ver, properties = Properties}) ->
     Size = maps:get('Maximum-Packet-Size', Properties, ?MAX_PACKET_SIZE),
@@ -534,15 +540,16 @@ mqtt_connect(State = #state{client_id   = ClientId,
                             will_flag   = WillFlag,
                             will_msg    = WillMsg,
                             properties  = Properties}) ->
-    ?WILL_MSG(WillQos, WillRetain, WillTopic, WillProps, WillPayload) = WillMsg,
+    ?WILL_MSG(WillQoS, WillRetain, WillTopic, WillProps, WillPayload) = WillMsg,
-    ConnProps = emqx_mqtt_properties:filter(?CONNECT, maps:to_list(Properties)),
+    ConnProps = emqx_mqtt_properties:filter(?CONNECT, Properties),
+    io:format("ConnProps: ~p~n", [ConnProps]),
     send(?CONNECT_PACKET(
             #mqtt_packet_connect{proto_ver    = ProtoVer,
                                  proto_name   = ProtoName,
                                  is_bridge    = IsBridge,
                                  clean_start  = CleanStart,
                                  will_flag    = WillFlag,
-                                 will_qos     = WillQos,
+                                 will_qos     = WillQoS,
                                  will_retain  = WillRetain,
                                  keepalive    = KeepAlive,
                                  properties   = ConnProps,
@@ -624,7 +631,7 @@ connected({call, From}, SubReq = {subscribe, Properties, Topics},
             {stop_and_reply, Reason, [{reply, From, Error}]}
     end;
 
-connected({call, From}, {publish, Msg = #mqtt_message{qos = ?QOS_0}}, State) ->
+connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) ->
     case send(Msg, State) of
         {ok, NewState} ->
             {keep_state, NewState, [{reply, From, ok}]};
@@ -632,14 +639,14 @@ connected({call, From}, {publish, Msg = #mqtt_message{qos = ?QOS_0}}, State) ->
             {stop_and_reply, Reason, [{reply, From, Error}]}
     end;
 
-connected({call, From}, {publish, Msg = #mqtt_message{qos = Qos}},
+connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}},
           State = #state{inflight = Inflight, last_packet_id = PacketId})
-    when (Qos =:= ?QOS_1); (Qos =:= ?QOS_2) ->
+    when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) ->
     case emqx_inflight:is_full(Inflight) of
         true ->
             {keep_state, State, [{reply, From, {error, inflight_full}}]};
         false ->
-            Msg1 = Msg#mqtt_message{packet_id = PacketId},
+            Msg1 = Msg#mqtt_msg{packet_id = PacketId},
             case send(Msg1, State) of
                 {ok, NewState} ->
                     Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight),
@@ -690,7 +697,7 @@ connected(cast, {pubcomp, PacketId, ReasonCode, Properties}, State) ->
     send_puback(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), State);
 
 connected(cast, Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId), State) ->
-    {keep_state, deliver_msg(packet_to_msg(Packet), State)};
+    {keep_state, deliver(packet_to_msg(Packet), State)};
 
 connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), State = #state{paused = true}) ->
     {keep_state, State};
@@ -698,7 +705,7 @@ connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), State = #state{paused = true})
 connected(cast, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId),
           State = #state{auto_ack = AutoAck}) ->
 
-    _ = deliver_msg(packet_to_msg(Packet), State),
+    _ = deliver(packet_to_msg(Packet), State),
     case AutoAck of
         true  -> send_puback(?PUBACK_PACKET(PacketId), State);
         false -> {keep_state, State}
@@ -716,7 +723,7 @@ connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, PacketId),
 connected(cast, ?PUBACK_PACKET(PacketId, ReasonCode, Properties),
           State = #state{owner = Owner, inflight = Inflight}) ->
     case emqx_inflight:lookup(PacketId, Inflight) of
-        {value, {publish, #mqtt_message{packet_id = PacketId}, _Ts}} ->
+        {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} ->
             Owner ! {puback, #{packet_id   => PacketId,
                                reason_code => ReasonCode,
                                properties  => Properties}},
@@ -745,8 +752,7 @@ connected(cast, ?PUBREL_PACKET(PacketId),
           State = #state{awaiting_rel = AwaitingRel, auto_ack = AutoAck}) ->
      case maps:take(PacketId, AwaitingRel) of
          {Packet, AwaitingRel1} ->
-             NewState = deliver_msg(packet_to_msg(Packet),
+             NewState = deliver(packet_to_msg(Packet), State#state{awaiting_rel = AwaitingRel1}),
-                                    State#state{awaiting_rel = AwaitingRel1}),
              case AutoAck of
                  true  -> send_puback(?PUBCOMP_PACKET(PacketId), NewState);
                  false -> {keep_state, NewState}
@@ -960,9 +966,9 @@ retry_send([{Type, Msg, Ts} | Msgs], Now, State = #state{retry_interval = Interv
         false -> {keep_state, ensure_retry_timer(Interval - Diff, State)}
     end.
 
-retry_send(publish, Msg = #mqtt_message{qos = QoS, packet_id = PacketId},
+retry_send(publish, Msg = #mqtt_msg{qos = QoS, packet_id = PacketId},
            Now, State = #state{inflight = Inflight}) ->
-    Msg1 = Msg#mqtt_message{dup = (QoS =:= ?QOS1)},
+    Msg1 = Msg#mqtt_msg{dup = (QoS =:= ?QOS1)},
     case send(Msg1, State) of
         {ok, NewState} ->
             Inflight1 = emqx_inflight:update(PacketId, {publish, Msg1, Now}, Inflight),
@@ -979,42 +985,29 @@ retry_send(pubrel, PacketId, Now, State = #state{inflight = Inflight}) ->
             Error
     end.
 
-deliver_msg(#mqtt_message{qos        = QoS,
+deliver(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId,
-                          dup        = Dup,
+                  topic = Topic, props = Props, payload = Payload},
-                          retain     = Retain,
-                          topic      = Topic,
-                          packet_id  = PacketId,
-                          properties = Properties,
-                          payload    = Payload},
         State = #state{owner = Owner}) ->
-    Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain,
+    Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId,
-                        packet_id => PacketId, topic => Topic,
+                        topic => Topic, properties => Props, payload => Payload}},
-                        properties => Properties, payload => Payload}},
     State.
 
-packet_to_msg(?PUBLISH_PACKET(Header, Topic, PacketId, Properties, Payload)) ->
+packet_to_msg(?PUBLISH_PACKET(Header, Topic, PacketId, Props, Payload)) ->
     #mqtt_packet_header{qos = QoS, retain = R, dup = Dup} = Header,
-    #mqtt_message{qos = QoS, retain = R, dup = Dup,
+    #mqtt_msg{qos = QoS, retain = R, dup = Dup, packet_id = PacketId,
-                  packet_id = PacketId, topic = Topic,
+              topic = Topic, props = Props, payload = Payload}.
-                  properties = Properties, payload = Payload}.
 
-msg_to_packet(#mqtt_message{qos        = Qos,
+msg_to_packet(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId,
-                            dup        = Dup,
+                       topic = Topic, props = Props, payload = Payload}) ->
-                            retain     = Retain,
-                            topic      = Topic,
-                            packet_id  = PacketId,
-                            properties = Properties,
-                            payload    = Payload}) ->
     #mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
-                                                qos    = Qos,
+                                                qos    = QoS,
                                                 retain = Retain,
                                                 dup    = Dup},
                  variable = #mqtt_packet_publish{topic_name = Topic,
                                                  packet_id  = PacketId,
-                                                 properties = Properties},
+                                                 properties = Props},
                  payload  = Payload}.
 
-
 %%------------------------------------------------------------------------------
 %% Socket Connect/Send
 
@@ -1040,7 +1033,7 @@ send_puback(Packet, State) ->
         {error, Reason} -> {stop, Reason}
     end.
 
-send(Msg, State) when is_record(Msg, mqtt_message) ->
+send(Msg, State) when is_record(Msg, mqtt_msg) ->
     send(msg_to_packet(Msg), State);
 
 send(Packet, State = #state{socket = Sock, proto_ver = Ver})

src/emqx_connection.erl

@@ -20,33 +20,51 @@
 -include("emqx_mqtt.hrl").
 -include("emqx_misc.hrl").
 
--import(proplists, [get_value/2, get_value/3]).
-
 -export([start_link/3]).
-%% Management and Monitor API
--export([info/1, stats/1, kick/1, clean_acl_cache/2]).
--export([set_rate_limit/2, get_rate_limit/1]).
-%% SUB/UNSUB Asynchronously. Called by plugins.
--export([subscribe/2, unsubscribe/2]).
-%% Get the session proc?
--export([session/1]).
 
-%% gen_server Function Exports
+-export([info/1, stats/1, kick/1]).
+-export([get_session/1]).
+-export([clean_acl_cache/1]).
+-export([get_rate_limit/1, set_rate_limit/2]).
+-export([get_pub_limit/1, set_pub_limit/2]).
+
+%% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3,
          terminate/2]).
 
-%% Unused fields: connname, peerhost, peerport
+-record(state, {
--record(state, {transport, socket, peername, conn_state, await_recv,
+          transport,    %% Network transport module
-                rate_limit, max_packet_size, proto_state, parse_state,
+          socket,       %% TCP or SSL Socket
-                keepalive, enable_stats, idle_timeout, force_gc_count}).
+          peername,     %% Peername of the socket
+          sockname,     %% Sockname of the socket
+          conn_state,   %% Connection state: running | blocked
+          await_recv,   %% Awaiting recv
+          incoming,     %% Incoming bytes and packets
+          pub_limit,    %% Publish rate limit
+          rate_limit,   %% Throughput rate limit
+          limit_timer,  %% Rate limit timer
+          proto_state,  %% MQTT protocol state
+          parse_state,  %% MQTT parse state
+          keepalive,    %% MQTT keepalive timer
+          enable_stats, %% Enable stats
+          stats_timer,  %% Stats timer
+          idle_timeout  %% Connection idle timeout
+         }).
+
+-define(INFO_KEYS, [peername, sockname, conn_state, await_recv, rate_limit, pub_limit]).
 
--define(INFO_KEYS, [peername, conn_state, await_recv]).
 -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
--define(LOG(Level, Format, Args, State),
-        emqx_logger:Level("Conn(~s): " ++ Format, [esockd_net:format(State#state.peername) | Args])).
 
-start_link(Transport, Sock, Options) ->
+-define(LOG(Level, Format, Args, State),
-    {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Sock, Options]])}.
+        emqx_logger:Level("Conn(~s): " ++ Format,
+                          [esockd_net:format(State#state.peername) | Args])).
+
+start_link(Transport, Socket, Options) ->
+    {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Socket, Options]])}.
+
+%%------------------------------------------------------------------------------
+%% API
+%%------------------------------------------------------------------------------
 
 info(CPid) ->
     gen_server:call(CPid, info).
@@ -57,152 +75,151 @@ stats(CPid) ->
 kick(CPid) ->
     gen_server:call(CPid, kick).
 
-set_rate_limit(CPid, Rl) ->
+get_session(CPid) ->
-    gen_server:call(CPid, {set_rate_limit, Rl}).
+    gen_server:call(CPid, session, infinity).
+
+clean_acl_cache(CPid) ->
+    gen_server:call(CPid, clean_acl_cache).
 
 get_rate_limit(CPid) ->
     gen_server:call(CPid, get_rate_limit).
 
-subscribe(CPid, TopicTable) ->
+set_rate_limit(CPid, Rl = {_Rate, _Burst}) ->
-    CPid ! {subscribe, TopicTable}.
+    gen_server:call(CPid, {set_rate_limit, Rl}).
 
-unsubscribe(CPid, Topics) ->
+get_pub_limit(CPid) ->
-    CPid ! {unsubscribe, Topics}.
+    gen_server:call(CPid, get_pub_limit).
 
-session(CPid) ->
+set_pub_limit(CPid, Rl = {_Rate, _Burst}) ->
-    gen_server:call(CPid, session, infinity).
+    gen_server:call(CPid, {set_pub_limit, Rl}).
-
-clean_acl_cache(CPid, Topic) ->
-    gen_server:call(CPid, {clean_acl_cache, Topic}).
 
 %%------------------------------------------------------------------------------
 %% gen_server callbacks
 %%------------------------------------------------------------------------------
 
-init([Transport, Sock, Options]) ->
+init([Transport, RawSocket, Options]) ->
-    case Transport:wait(Sock) of
+    case Transport:wait(RawSocket) of
-        {ok, NewSock} ->
+        {ok, Socket} ->
-            {ok, Peername} = Transport:ensure_ok_or_exit(peername, [NewSock]),
+            {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
-            do_init(Transport, Sock, Peername, Options);
+            {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
-        {error, Reason} ->
+            Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
-            {stop, Reason}
+            Zone = proplists:get_value(zone, Options),
-    end.
+            RateLimit = init_rate_limit(emqx_zone:get_env(Zone, rate_limit)),
-
+            PubLimit = init_rate_limit(emqx_zone:get_env(Zone, publish_limit)),
-do_init(Transport, Sock, Peername, Options) ->
+            EnableStats = emqx_zone:get_env(Zone, enable_stats, false),
-    io:format("Options: ~p~n", [Options]),
+            IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000),
-    RateLimit = get_value(rate_limit, Options),
+            SendFun = send_fun(Transport, Socket, Peername),
-    PacketSize = get_value(max_packet_size, Options, ?MAX_PACKET_SIZE),
+            ProtoState = emqx_protocol:init(#{zone     => Zone,
-    SendFun = send_fun(Transport, Sock, Peername),
+                                              peername => Peername,
-    ProtoState = emqx_protocol:init(Transport, Sock, Peername, SendFun, Options),
+                                              sockname => Sockname,
-    EnableStats = get_value(client_enable_stats, Options, false),
+                                              peercert => Peercert,
-    IdleTimout = get_value(client_idle_timeout, Options, 30000),
+                                              sendfun  => SendFun}, Options),
-    ForceGcCount = emqx_gc:conn_max_gc_count(),
+            ParseState = emqx_protocol:parser(ProtoState),
             State = run_socket(#state{transport    = Transport,
-                              socket          = Sock,
+                                      socket       = Socket,
                                       peername     = Peername,
                                       await_recv   = false,
                                       conn_state   = running,
                                       rate_limit   = RateLimit,
-                              max_packet_size = PacketSize,
+                                      pub_limit    = PubLimit,
                                       proto_state  = ProtoState,
+                                      parse_state  = ParseState,
                                       enable_stats = EnableStats,
-                              idle_timeout    = IdleTimout,
+                                      idle_timeout = IdleTimout}),
-                              force_gc_count  = ForceGcCount}),
             gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}],
-                          init_parse_state(State), self(), IdleTimout).
+                                  State, self(), IdleTimout);
+        {error, Reason} ->
+            {stop, Reason}
+    end.
 
-send_fun(Transport, Sock, Peername) ->
+init_rate_limit(undefined) ->
-    Self = self(),
+    undefined;
-    fun(Packet) ->
+init_rate_limit({Rate, Burst}) ->
-        Data = emqx_frame:serialize(Packet),
+    esockd_rate_limit:new(Rate, Burst).
+
+send_fun(Transport, Socket, Peername) ->
+    fun(Data) ->
+        try Transport:async_send(Socket, Data) of
+            ok  ->
                 ?LOG(debug, "SEND ~p", [Data], #state{peername = Peername}),
-        emqx_metrics:inc('bytes/sent', iolist_size(Data)),
+                emqx_metrics:inc('bytes/sent', iolist_size(Data)), ok;
-        try Transport:async_send(Sock, Data) of
+            Error -> Error
-            ok -> ok;
-            {error, Reason} -> Self ! {shutdown, Reason}
         catch
-            error:Error -> Self ! {shutdown, Error}
+            error:Error -> {error, Error}
         end
     end.
 
-init_parse_state(State = #state{max_packet_size = Size, proto_state = ProtoState}) ->
+handle_call(info, From, State = #state{transport = Transport, socket = Socket, proto_state = ProtoState}) ->
-    Version = emqx_protocol:get(proto_ver, ProtoState),
-    State#state{parse_state = emqx_frame:initial_state(#{max_packet_size => Size, version => Version})}.
-
-handle_call(info, From, State = #state{proto_state = ProtoState}) ->
     ProtoInfo = emqx_protocol:info(ProtoState),
-    ClientInfo = ?record_to_proplist(state, State, ?INFO_KEYS),
+    ConnInfo = [{socktype, Transport:type(Socket)}
-    {reply, Stats, _, _} = handle_call(stats, From, State),
+                | ?record_to_proplist(state, State, ?INFO_KEYS)],
-    reply(lists:append([ClientInfo, ProtoInfo, Stats]), State);
+    StatsInfo = element(2, handle_call(stats, From, State)),
+    {reply, lists:append([ConnInfo, StatsInfo, ProtoInfo]), State};
 
-handle_call(stats, _From, State = #state{proto_state = ProtoState}) ->
+handle_call(stats, _From, State = #state{transport = Transport, socket = Sock, proto_state = ProtoState}) ->
-    reply(lists:append([emqx_misc:proc_stats(),
+    ProcStats = emqx_misc:proc_stats(),
-                        emqx_protocol:stats(ProtoState),
+    ProtoStats = emqx_protocol:stats(ProtoState),
-                        sock_stats(State)]), State);
+    SockStats = case Transport:getstat(Sock, ?SOCK_STATS) of
+                    {ok, Ss}   -> Ss;
+                    {error, _} -> []
+                end,
+    {reply, lists:append([ProcStats, ProtoStats, SockStats]), State};
 
 handle_call(kick, _From, State) ->
     {stop, {shutdown, kick}, ok, State};
 
-handle_call({set_rate_limit, Rl}, _From, State) ->
+handle_call(get_session, _From, State = #state{proto_state = ProtoState}) ->
-    reply(ok, State#state{rate_limit = Rl});
+    {reply, emqx_protocol:session(ProtoState), State};
+
+handle_call(clean_acl_cache, _From, State = #state{proto_state = ProtoState}) ->
+    {reply, ok, State#state{proto_state = emqx_protocol:clean_acl_cache(ProtoState)}};
 
 handle_call(get_rate_limit, _From, State = #state{rate_limit = Rl}) ->
-    reply(Rl, State);
+    {reply, esockd_rate_limit:info(Rl), State};
 
-handle_call(session, _From, State = #state{proto_state = ProtoState}) ->
+handle_call({set_rate_limit, {Rate, Burst}}, _From, State) ->
-    reply(emqx_protocol:session(ProtoState), State);
+    {reply, ok, State#state{rate_limit = esockd_rate_limit:new(Rate, Burst)}};
 
-handle_call({clean_acl_cache, Topic}, _From, State) ->
+handle_call(get_publish_limit, _From, State = #state{pub_limit = Rl}) ->
-    erase({acl, publish, Topic}),
+    {reply, esockd_rate_limit:info(Rl), State};
-    reply(ok, State);
+
+handle_call({set_publish_limit, {Rate, Burst}}, _From, State) ->
+    {reply, ok, State#state{pub_limit = esockd_rate_limit:new(Rate, Burst)}};
 
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected Call: ~p", [Req], State),
+    ?LOG(error, "unexpected call: ~p", [Req], State),
-    {reply, ignore, State}.
+    {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected Cast: ~p", [Msg], State),
+    ?LOG(error, "unexpected cast: ~p", [Msg], State),
     {noreply, State}.
 
-handle_info({subscribe, TopicTable}, State) ->
+handle_info(SubReq = {subscribe, _TopicTable}, State) ->
     with_proto(
       fun(ProtoState) ->
-          emqx_protocol:subscribe(TopicTable, ProtoState)
+          emqx_protocol:process(SubReq, ProtoState)
       end, State);
 
-handle_info({unsubscribe, Topics}, State) ->
+handle_info(UnsubReq = {unsubscribe, _Topics}, State) ->
     with_proto(
       fun(ProtoState) ->
-          emqx_protocol:unsubscribe(Topics, ProtoState)
+          emqx_protocol:process(UnsubReq, ProtoState)
       end, State);
 
-%% Asynchronous SUBACK
+handle_info({deliver, PubOrAck}, State) ->
-handle_info({suback, PacketId, GrantedQos}, State) ->
     with_proto(
       fun(ProtoState) ->
-          Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
+          emqx_protocol:deliver(PubOrAck, ProtoState)
-          emqx_protocol:send(Packet, ProtoState)
+      end, maybe_gc(ensure_stats_timer(State)));
-      end, State);
 
-handle_info({deliver, Message}, State) ->
+handle_info(emit_stats, State = #state{proto_state = ProtoState}) ->
-    with_proto(
+    Stats = element(2, handle_call(stats, undefined, State)),
-      fun(ProtoState) ->
+    emqx_cm:set_client_stats(emqx_protocol:clientid(ProtoState), Stats),
-          emqx_protocol:send(Message, ProtoState)
+    {noreply, State = #state{stats_timer = undefined}, hibernate};
-      end, State);
-
-handle_info({redeliver, {?PUBREL, PacketId}}, State) ->
-    with_proto(
-      fun(ProtoState) ->
-          emqx_protocol:pubrel(PacketId, ProtoState)
-      end, State);
-
-handle_info(emit_stats, State) ->
-    {noreply, emit_stats(State), hibernate};
 
 handle_info(timeout, State) ->
     shutdown(idle_timeout, State);
 
-%% Fix issue #535
 handle_info({shutdown, Error}, State) ->
     shutdown(Error, State);
 
@@ -211,25 +228,25 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) ->
     shutdown(conflict, State);
 
 handle_info(activate_sock, State) ->
-    {noreply, run_socket(State#state{conn_state = running})};
+    {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})};
 
 handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) ->
     Size = iolist_size(Data),
     ?LOG(debug, "RECV ~p", [Data], State),
     emqx_metrics:inc('bytes/received', Size),
-    received(Data, rate_limit(Size, State#state{await_recv = false}));
+    Incoming = #{bytes => Size, packets => 0},
+    handle_packet(Data, State#state{await_recv = false, incoming = Incoming});
 
 handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) ->
     shutdown(Reason, State);
 
 handle_info({inet_reply, _Sock, ok}, State) ->
-    {noreply, gc(State)}; %% Tune GC
+    {noreply, State};
 
 handle_info({inet_reply, _Sock, {error, Reason}}, State) ->
     shutdown(Reason, State);
 
-handle_info({keepalive, start, Interval},
+handle_info({keepalive, start, Interval}, State = #state{transport = Transport, socket = Sock}) ->
-            State = #state{transport = Transport, socket = Sock}) ->
     ?LOG(debug, "Keepalive at the interval of ~p", [Interval], State),
     StatFun = fun() ->
                 case Transport:getstat(Sock, [recv_oct]) of
@@ -258,20 +275,18 @@ handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) ->
     end;
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected Info: ~p", [Info], State),
+    ?LOG(error, "unexpected info: ~p", [Info], State),
     {noreply, State}.
 
 terminate(Reason, State = #state{transport   = Transport,
                                  socket      = Sock,
                                  keepalive   = KeepAlive,
                                  proto_state = ProtoState}) ->
-
     ?LOG(debug, "Terminated for ~p", [Reason], State),
     Transport:fast_close(Sock),
     emqx_keepalive:cancel(KeepAlive),
     case {ProtoState, Reason} of
-        {undefined, _} ->
+        {undefined, _} -> ok;
-            ok;
         {_, {shutdown, Error}} ->
             emqx_protocol:shutdown(Error, ProtoState);
         {_, Reason} ->
@@ -281,25 +296,29 @@ terminate(Reason, State = #state{transport   = Transport,
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Internal functions
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
-%% Receive and Parse TCP Data
+%% Receive and parse TCP data
-received(<<>>, State) ->
+handle_packet(<<>>, State) ->
-    {noreply, gc(State)};
+    {noreply, maybe_gc(ensure_stats_timer(ensure_rate_limit(State)))};
 
-received(Bytes, State = #state{parse_state  = ParseState,
+handle_packet(Bytes, State = #state{incoming     = Incoming,
+                                    parse_state  = ParseState,
                                     proto_state  = ProtoState,
                                     idle_timeout = IdleTimeout}) ->
     case catch emqx_frame:parse(Bytes, ParseState) of
         {more, NewParseState} ->
             {noreply, State#state{parse_state = NewParseState}, IdleTimeout};
-        {ok, Packet, Rest} ->
+        {ok, Packet = ?PACKET(Type), Rest} ->
             emqx_metrics:received(Packet),
             case emqx_protocol:received(Packet, ProtoState) of
                 {ok, ProtoState1} ->
-                    received(Rest, init_parse_state(State#state{proto_state = ProtoState1}));
+                    ParseState1 = emqx_protocol:parser(ProtoState1),
+                    handle_packet(Rest, State#state{incoming    = count_packets(Type, Incoming),
+                                                    proto_state = ProtoState1,
+                                                    parse_state = ParseState1});
                 {error, Error} ->
                     ?LOG(error, "Protocol error - ~p", [Error], State),
                     shutdown(Error, State);
@@ -312,21 +331,32 @@ received(Bytes, State = #state{parse_state  = ParseState,
             ?LOG(error, "Framing error - ~p", [Error], State),
             shutdown(Error, State);
         {'EXIT', Reason} ->
-            ?LOG(error, "Parser failed for ~p", [Reason], State),
+            ?LOG(error, "Parse failed for ~p~nError data:~p", [Reason, Bytes], State),
-            ?LOG(error, "Error data: ~p", [Bytes], State),
+            shutdown(parse_error, State)
-            shutdown(parser_error, State)
     end.
 
-rate_limit(_Size, State = #state{rate_limit = undefined}) ->
+count_packets(?PUBLISH, Incoming = #{packets := Num}) ->
+    Incoming#{packets := Num + 1};
+count_packets(?SUBSCRIBE, Incoming = #{packets := Num}) ->
+    Incoming#{packets := Num + 1};
+count_packets(_Type, Incoming) ->
+    Incoming.
+
+ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl,
+                                 incoming = #{bytes := Bytes, packets := Pkts}}) ->
+    ensure_rate_limit([{Pl, #state.pub_limit, Pkts}, {Rl, #state.rate_limit, Bytes}], State).
+
+ensure_rate_limit([], State) ->
     run_socket(State);
-rate_limit(Size, State = #state{rate_limit = Rl}) ->
+ensure_rate_limit([{undefined, _Pos, _Num}|Limiters], State) ->
-    case Rl:check(Size) of
+    ensure_rate_limit(Limiters, State);
+ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) ->
+   case esockd_rate_limit:check(Num, Rl) of
        {0, Rl1} ->
-            run_socket(State#state{conn_state = running, rate_limit = Rl1});
+           ensure_rate_limit(Limiters, setelement(Pos, State, Rl1));
        {Pause, Rl1} ->
-            ?LOG(warning, "Rate limiter pause for ~p", [Pause], State),
+           TRef = erlang:send_after(Pause, self(), activate_sock),
-            erlang:send_after(Pause, self(), activate_sock),
+           setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1)
-            State#state{conn_state = blocked, rate_limit = Rl1}
    end.
 
 run_socket(State = #state{conn_state = blocked}) ->
@@ -338,29 +368,21 @@ run_socket(State = #state{transport = Transport, socket = Sock}) ->
     State#state{await_recv = true}.
 
 with_proto(Fun, State = #state{proto_state = ProtoState}) ->
-    {ok, ProtoState1} = Fun(ProtoState),
+    case Fun(ProtoState) of
-    {noreply, State#state{proto_state = ProtoState1}}.
+        {ok, ProtoState1} ->
-
+            {noreply, State#state{proto_state = ProtoState1}};
-emit_stats(State = #state{proto_state = ProtoState}) ->
+        {error, Reason} ->
-    emit_stats(emqx_protocol:clientid(ProtoState), State).
+            shutdown(Reason, State);
-
+        {error, Reason, ProtoState1} ->
-emit_stats(_ClientId, State = #state{enable_stats = false}) ->
+            shutdown(Reason, State#state{proto_state = ProtoState1})
-    State;
-emit_stats(undefined, State) ->
-    State;
-emit_stats(ClientId, State) ->
-    {reply, Stats, _, _} = handle_call(stats, undefined, State),
-    emqx_cm:set_client_stats(ClientId, Stats),
-    State.
-
-sock_stats(#state{transport = Transport, socket = Sock}) ->
-    case Transport:getstat(Sock, ?SOCK_STATS) of
-        {ok, Ss} -> Ss;
-        _Error   -> []
     end.
 
-reply(Reply, State) ->
+ensure_stats_timer(State = #state{enable_stats = true,
-    {reply, Reply, State, hibernate}.
+                                  stats_timer  = undefined,
+                                  idle_timeout = IdleTimeout}) ->
+    State#state{stats_timer = erlang:send_after(IdleTimeout, self(), emit_stats)};
+ensure_stats_timer(State) ->
+    State.
 
 shutdown(Reason, State) ->
     stop({shutdown, Reason}, State).
@@ -368,7 +390,8 @@ shutdown(Reason, State) ->
 stop(Reason, State) ->
     {stop, Reason, State}.
 
-gc(State = #state{transport = Transport, socket = Sock}) ->
+maybe_gc(State) ->
-    Cb = fun() -> Transport:gc(Sock), emit_stats(State) end,
+    State. %% TODO:...
-    emqx_gc:maybe_force_gc(#state.force_gc_count, State, Cb).
+    %%Cb = fun() -> Transport:gc(Sock), end,
+    %%emqx_gc:maybe_force_gc(#state.force_gc_count, State, Cb).
 

src/emqx_frame.erl

@@ -121,7 +121,7 @@ parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) ->
     <<UsernameFlag : 1,
       PasswordFlag : 1,
       WillRetain   : 1,
-      WillQos      : 2,
+      WillQoS      : 2,
       WillFlag     : 1,
       CleanStart   : 1,
       _Reserved    : 1,
@@ -138,7 +138,7 @@ parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) ->
                                       is_bridge    = (BridgeTag =:= 8),
                                       clean_start  = bool(CleanStart),
                                       will_flag    = bool(WillFlag),
-                                      will_qos     = WillQos,
+                                      will_qos     = WillQoS,
                                       will_retain  = bool(WillRetain),
                                       keepalive    = KeepAlive,
                                       properties   = Properties,
@@ -242,6 +242,9 @@ parse_packet_id(<<PacketId:16/big, Rest/binary>>) ->
 
 parse_properties(Bin, Ver) when Ver =/= ?MQTT_PROTO_V5 ->
     {undefined, Bin};
+%% TODO: version mess?
+parse_properties(<<>>, ?MQTT_PROTO_V5) ->
+    {#{}, <<>>};
 parse_properties(<<0, Rest/binary>>, ?MQTT_PROTO_V5) ->
     {#{}, Rest};
 parse_properties(Bin, ?MQTT_PROTO_V5) ->
@@ -382,7 +385,7 @@ serialize_variable(#mqtt_packet_connect{
                       is_bridge    = IsBridge,
                       clean_start  = CleanStart,
                       will_flag    = WillFlag,
-                      will_qos     = WillQos,
+                      will_qos     = WillQoS,
                       will_retain  = WillRetain,
                       keepalive    = KeepAlive,
                       properties   = Properties,
@@ -400,7 +403,7 @@ serialize_variable(#mqtt_packet_connect{
        (flag(Username)):1,
        (flag(Password)):1,
        (flag(WillRetain)):1,
-       WillQos:2,
+       WillQoS:2,
        (flag(WillFlag)):1,
        (flag(CleanStart)):1,
        0:1,

src/emqx_kernel_sup.erl

@@ -31,6 +31,7 @@ init([]) ->
            child_spec(emqx_stats, worker),
            child_spec(emqx_metrics, worker),
            child_spec(emqx_ctl, worker),
+           child_spec(emqx_zone, worker),
            child_spec(emqx_tracer, worker)]}}.
 
 child_spec(M, worker) ->

src/emqx_listeners.erl

@@ -33,7 +33,7 @@ start_listener({tcp, ListenOn, Options}) ->
     start_mqtt_listener('mqtt:tcp', ListenOn, Options);
 %% Start MQTT/TLS listener
 start_listener({Proto, ListenOn, Options}) when Proto == ssl; Proto == tls ->
-    start_mqtt_listener('mqtt:tls', ListenOn, Options);
+    start_mqtt_listener('mqtt:ssl', ListenOn, Options);
 %% Start MQTT/WS listener
 start_listener({Proto, ListenOn, Options}) when Proto == http; Proto == ws ->
     start_http_listener('mqtt:ws', ListenOn, Options);

src/emqx_message.erl

@@ -17,45 +17,43 @@
 -include("emqx.hrl").
 -include("emqx_mqtt.hrl").
 
--export([new/2, new/3, new/4, new/5]).
+-export([make/2, make/3, make/4]).
+-export([set_flags/2]).
 -export([get_flag/2, get_flag/3, set_flag/2, set_flag/3, unset_flag/2]).
+-export([set_headers/2]).
 -export([get_header/2, get_header/3, set_header/3]).
--export([get_user_property/2, get_user_property/3, set_user_property/3]).
 
--spec(new(topic(), payload()) -> message()).
+-spec(make(topic(), payload()) -> message()).
-new(Topic, Payload) ->
+make(Topic, Payload) ->
-    new(undefined, Topic, Payload).
+    make(undefined, Topic, Payload).
 
--spec(new(atom() | client(), topic(), payload()) -> message()).
+-spec(make(atom() | client_id(), topic(), payload()) -> message()).
-new(From, Topic, Payload) when is_atom(From); is_record(From, client) ->
+make(From, Topic, Payload) ->
-    new(From, #{qos => ?QOS0}, Topic, Payload).
+    make(From, ?QOS0, Topic, Payload).
 
--spec(new(atom() | client(), message_flags(), topic(), payload()) -> message()).
+-spec(make(atom() | client_id(), qos(), topic(), payload()) -> message()).
-new(From, Flags, Topic, Payload) when is_atom(From); is_record(From, client) ->
+make(From, QoS, Topic, Payload) ->
-    new(From, Flags, #{}, Topic, Payload).
+    #message{id         = msgid(QoS),
-
+             qos        = QoS,
--spec(new(atom() | client(), message_flags(), message_headers(), topic(), payload()) -> message()).
-new(From, Flags, Headers, Topic, Payload) when is_atom(From); is_record(From, client) ->
-    #message{id         = msgid(),
-             qos        = ?QOS0,
              from       = From,
-             sender     = self(),
+             flags      = #{dup => false},
-             flags      = Flags,
-             headers    = Headers,
              topic      = Topic,
-             properties = #{},
              payload    = Payload,
              timestamp  = os:timestamp()}.
 
-msgid() -> emqx_guid:gen().
+msgid(?QOS0) -> undefined;
+msgid(_QoS)  -> emqx_guid:gen().
+
+set_flags(Flags, Msg = #message{flags = undefined}) when is_map(Flags) ->
+    Msg#message{flags = Flags};
+set_flags(New, Msg = #message{flags = Old}) when is_map(New) ->
+    Msg#message{flags = maps:merge(Old, New)}.
 
-%% @doc Get flag
 get_flag(Flag, Msg) ->
     get_flag(Flag, Msg, false).
 get_flag(Flag, #message{flags = Flags}, Default) ->
     maps:get(Flag, Flags, Default).
 
-%% @doc Set flag
 -spec(set_flag(message_flag(), message()) -> message()).
 set_flag(Flag, Msg = #message{flags = Flags}) when is_atom(Flag) ->
     Msg#message{flags = maps:put(Flag, true, Flags)}.
@@ -64,27 +62,22 @@ set_flag(Flag, Msg = #message{flags = Flags}) when is_atom(Flag) ->
 set_flag(Flag, Val, Msg = #message{flags = Flags}) when is_atom(Flag) ->
     Msg#message{flags = maps:put(Flag, Val, Flags)}.
 
-%% @doc Unset flag
 -spec(unset_flag(message_flag(), message()) -> message()).
 unset_flag(Flag, Msg = #message{flags = Flags}) ->
     Msg#message{flags = maps:remove(Flag, Flags)}.
 
-%% @doc Get header
+set_headers(Headers, Msg = #message{headers = undefined}) when is_map(Headers) ->
+    Msg#message{headers = Headers};
+set_headers(New, Msg = #message{headers = Old}) when is_map(New) ->
+    Msg#message{headers = maps:merge(Old, New)}.
+
 get_header(Hdr, Msg) ->
     get_header(Hdr, Msg, undefined).
 get_header(Hdr, #message{headers = Headers}, Default) ->
     maps:get(Hdr, Headers, Default).
 
-%% @doc Set header
+set_header(Hdr, Val, Msg = #message{headers = undefined}) ->
+    Msg#message{headers = #{Hdr => Val}};
 set_header(Hdr, Val, Msg = #message{headers = Headers}) ->
     Msg#message{headers = maps:put(Hdr, Val, Headers)}.
 
-%% @doc Get user property
-get_user_property(Key, Msg) ->
-    get_user_property(Key, Msg, undefined).
-get_user_property(Key, #message{properties = Props}, Default) ->
-    maps:get(Key, Props, Default).
-
-set_user_property(Key, Val, Msg = #message{properties = Props}) ->
-    Msg#message{properties = maps:put(Key, Val, Props)}.
-

src/emqx_metrics.erl

@@ -171,10 +171,10 @@ update_counter(Key, UpOp) ->
 received(Packet) ->
     inc('packets/received'),
     received1(Packet).
-received1(?PUBLISH_PACKET(Qos, _PktId)) ->
+received1(?PUBLISH_PACKET(QoS, _PktId)) ->
     inc('packets/publish/received'),
     inc('messages/received'),
-    qos_received(Qos);
+    qos_received(QoS);
 received1(?PACKET(Type)) ->
     received2(Type).
 received2(?CONNECT) ->
@@ -206,15 +206,15 @@ qos_received(?QOS_2) ->
 
 %% @doc Count packets received. Will not count $SYS PUBLISH.
 -spec(sent(mqtt_packet()) -> ignore | non_neg_integer()).
-sent(?PUBLISH_PACKET(_Qos, <<"$SYS/", _/binary>>, _, _)) ->
+sent(?PUBLISH_PACKET(_QoS, <<"$SYS/", _/binary>>, _, _)) ->
     ignore;
 sent(Packet) ->
     inc('packets/sent'),
     sent1(Packet).
-sent1(?PUBLISH_PACKET(Qos, _PktId)) ->
+sent1(?PUBLISH_PACKET(QoS, _PktId)) ->
     inc('packets/publish/sent'),
     inc('messages/sent'),
-    qos_sent(Qos);
+    qos_sent(QoS);
 sent1(?PACKET(Type)) ->
     sent2(Type).
 sent2(?CONNACK) ->

src/emqx_mod_presence.erl

@@ -39,8 +39,7 @@ on_client_connected(ConnAck, Client = #client{id = ClientId,
                                 {connack, ConnAck},
                                 {ts, emqx_time:now_secs()}]) of
         {ok, Payload} ->
-            Msg = message(qos(Env), topic(connected, ClientId), Payload),
+            emqx:publish(message(qos(Env), topic(connected, ClientId), Payload));
-            emqx:publish(emqx_message:set_flag(sys, Msg));
         {error, Reason} ->
             emqx_logger:error("[Presence Module] Json error: ~p", [Reason])
     end,
@@ -52,8 +51,7 @@ on_client_disconnected(Reason, #client{id = ClientId, username = Username}, Env)
                                 {reason, reason(Reason)},
                                 {ts, emqx_time:now_secs()}]) of
         {ok, Payload} ->
-            Msg = message(qos(Env), topic(disconnected, ClientId), Payload),
+            emqx_broker:publish(message(qos(Env), topic(disconnected, ClientId), Payload));
-            emqx:publish(emqx_message:set_flag(sys, Msg));
         {error, Reason} ->
             emqx_logger:error("[Presence Module] Json error: ~p", [Reason])
     end, ok.
@@ -62,9 +60,9 @@ unload(_Env) ->
     emqx:unhook('client.connected',    fun ?MODULE:on_client_connected/3),
     emqx:unhook('client.disconnected', fun ?MODULE:on_client_disconnected/3).
 
-message(Qos, Topic, Payload) ->
+message(QoS, Topic, Payload) ->
-    Msg = emqx_message:make(?MODULE, Topic, iolist_to_binary(Payload)),
+    Msg = emqx_message:make(?MODULE, QoS, Topic, iolist_to_binary(Payload)),
-    emqx_message:set_header(qos, Qos, Msg).
+    emqx_message:set_flags(#{sys => true}, Msg).
 
 topic(connected, ClientId) ->
     emqx_topic:systop(iolist_to_binary(["clients/", ClientId, "/connected"]));

src/emqx_mod_subscription.erl

@@ -34,7 +34,7 @@ load(Topics) ->
 on_client_connected(RC, Client = #client{id = ClientId, pid = ClientPid, username = Username}, Topics)
     when RC < 16#80 ->
     Replace = fun(Topic) -> rep(<<"%u">>, Username, rep(<<"%c">>, ClientId, Topic)) end,
-    TopicTable = [{Replace(Topic), Qos} || {Topic, Qos} <- Topics],
+    TopicTable = [{Replace(Topic), QoS} || {Topic, QoS} <- Topics],
     ClientPid ! {subscribe, TopicTable},
     {ok, Client};
 

src/emqx_mqtt_properties.erl

@@ -104,17 +104,20 @@ id('Wildcard-Subscription-Available')   -> 16#28;
 id('Subscription-Identifier-Available') -> 16#29;
 id('Shared-Subscription-Available')     -> 16#2A.
 
-filter(Packet, Props) when ?CONNECT =< Packet, Packet =< ?AUTH ->
+filter(PacketType, Props) when is_map(Props) ->
-    Fun = fun(Name) ->
+    maps:from_list(filter(PacketType, maps:to_list(Props)));
+
+filter(PacketType, Props) when ?CONNECT =< PacketType, PacketType =< ?AUTH, is_list(Props) ->
+    Filter = fun(Name) ->
                  case maps:find(id(Name), ?PROPS_TABLE) of
                      {ok, {Name, _Type, 'ALL'}} ->
                          true;
-                {ok, {Name, _Type, Packets}} ->
+                     {ok, {Name, _Type, AllowedTypes}} ->
-                    lists:member(Packet, Packets);
+                         lists:member(PacketType, AllowedTypes);
                      error -> false
                  end
              end,
-    [Prop || Prop = {Name, _} <- Props, Fun(Name)].
+    [Prop || Prop = {Name, _} <- Props, Filter(Name)].
 
 validate(Props) when is_map(Props) ->
     lists:foreach(fun validate_prop/1, maps:to_list(Props)).

src/emqx_mqueue.erl

@@ -150,7 +150,7 @@ stats(#mqueue{type = Type, q = Q, max_len = MaxLen, len = Len, dropped = Dropped
 
 %% @doc Enqueue a message.
 -spec(in(message(), mqueue()) -> mqueue()).
-in(#message{qos = ?QOS_0}, MQ = #mqueue{qos0 = false}) ->
+in(#message{flags = #{qos := ?QOS_0}}, MQ = #mqueue{qos0 = false}) ->
     MQ;
 in(Msg, MQ = #mqueue{type = simple, q = Q, len = Len, max_len = 0}) ->
     MQ#mqueue{q = queue:in(Msg, Q), len = Len + 1};

src/emqx_packet.erl

@@ -15,12 +15,11 @@
 -module(emqx_packet).
 
 -include("emqx.hrl").
-
 -include("emqx_mqtt.hrl").
 
 -export([protocol_name/1, type_name/1]).
 -export([format/1]).
--export([to_message/1, from_message/1]).
+-export([to_message/2, from_message/2]).
 
 %% @doc Protocol name of version
 -spec(protocol_name(mqtt_version()) -> binary()).
@@ -34,43 +33,40 @@ type_name(Type) when Type > ?RESERVED andalso Type =< ?AUTH ->
     lists:nth(Type, ?TYPE_NAMES).
 
 %% @doc From Message to Packet
--spec(from_message(message()) -> mqtt_packet()).
+-spec(from_message(mqtt_packet_id(), message()) -> mqtt_packet()).
-from_message(Msg = #message{topic = Topic, payload = Payload}) ->
+from_message(PacketId, Msg = #message{qos = QoS, topic = Topic, payload = Payload}) ->
-    Qos = emqx_message:get_flag(qos, Msg, 0),
     Dup = emqx_message:get_flag(dup, Msg, false),
     Retain = emqx_message:get_flag(retain, Msg, false),
-    PacketId = emqx_message:get_header(packet_id, Msg),
     #mqtt_packet{header = #mqtt_packet_header{type   = ?PUBLISH,
-                                              qos    = Qos,
+                                              qos    = QoS,
                                               retain = Retain,
                                               dup    = Dup},
-                 variable = #mqtt_packet_publish{topic_name = Topic,
-                                                 packet_id  = PacketId},
-                 payload = Payload}.
-
-%% @doc Message from Packet
--spec(to_message(mqtt_packet()) -> message()).
-to_message(#mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
-                                                       retain = Retain,
-                                                       qos    = Qos,
-                                                       dup    = Dup},
                  variable = #mqtt_packet_publish{topic_name = Topic,
                                                  packet_id  = PacketId,
-                                                        properties = Properties},
+                                                 properties = #{}}, %%TODO:
-                        payload  = Payload}) ->
+                 payload = Payload}.
-    Flags = #{dup => Dup, retain => Retain, qos => Qos},
-    Msg = emqx_message:new(undefined, Flags, #{packet_id => PacketId}, Topic, Payload),
-    Msg#message{properties = Properties};
 
-to_message(#mqtt_packet_connect{will_flag = false}) ->
+%% @doc Message from Packet
+-spec(to_message(client_id(), mqtt_packet()) -> message()).
+to_message(ClientId, #mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
+                                                                 retain = Retain,
+                                                                 qos    = QoS,
+                                                                 dup    = Dup},
+                                  variable = #mqtt_packet_publish{topic_name = Topic,
+                                                                  properties = Props},
+                                  payload  = Payload}) ->
+    Msg = emqx_message:make(ClientId, QoS, Topic, Payload),
+    Msg#message{flags = #{dup => Dup, retain => Retain}, headers = Props};
+
+to_message(_ClientId, #mqtt_packet_connect{will_flag = false}) ->
     undefined;
-to_message(#mqtt_packet_connect{will_retain  = Retain,
+to_message(ClientId, #mqtt_packet_connect{will_retain  = Retain,
-                                will_qos     = Qos,
+                                          will_qos     = QoS,
                                           will_topic   = Topic,
                                           will_props   = Props,
                                           will_payload = Payload}) ->
-    Msg = emqx_message:new(undefined, #{qos => Qos, retain => Retain}, Topic, Payload),
+    Msg = emqx_message:make(ClientId, QoS, Topic, Payload),
-    Msg#message{properties = Props}.
+    Msg#message{flags = #{qos => QoS, retain => Retain}, headers = Props}.
 
 %% @doc Format packet
 -spec(format(mqtt_packet()) -> iolist()).
@@ -153,3 +149,4 @@ format_password(_Password) -> '******'.
 i(true)  -> 1;
 i(false) -> 0;
 i(I) when is_integer(I) -> I.
+

src/emqx_protocol.erl

@@ -18,108 +18,86 @@
 -include("emqx_mqtt.hrl").
 -include("emqx_misc.hrl").
 
--import(proplists, [get_value/2, get_value/3]).
+-export([init/2, info/1, stats/1, clientid/1, session/1]).
-
+-export([parser/1]).
-%% API
+-export([received/2, process/2, deliver/2, send/2]).
--export([init/3, init/5, get/2, info/1, stats/1, clientid/1, client/1, session/1]).
+-export([shutdown/2]).
--export([subscribe/2, unsubscribe/2, pubrel/2, shutdown/2]).
--export([received/2, send/2]).
--export([process/2]).
 
 -ifdef(TEST).
 -compile(export_all).
 -endif.
 
--record(proto_stats, {enable_stats = false, recv_pkt = 0, recv_msg = 0, send_pkt = 0, send_msg = 0}).
+-define(CAPABILITIES, [{max_packet_size,  ?MAX_PACKET_SIZE},
+                       {max_clientid_len, ?MAX_CLIENTID_LEN},
+                       {max_topic_alias,  0},
+                       {max_qos_allowed,  ?QOS2},
+                       {retain_available, true},
+                       {shared_subscription,   true},
+                       {wildcard_subscription, true}]).
 
-%% Protocol State
+-record(proto_state, {sockprops, capabilities, connected, client_id, client_pid,
-%% ws_initial_headers: Headers from first HTTP request for WebSocket Client.
+                      clean_start, proto_ver, proto_name, username, connprops,
--record(proto_state, {peername, sendfun, connected = false, client_id, client_pid,
+                      is_superuser, will_msg, keepalive, keepalive_backoff, session,
-                      clean_start, proto_ver, proto_name, username, is_superuser,
+                      recv_pkt = 0, recv_msg = 0, send_pkt = 0, send_msg = 0,
-                      will_msg, keepalive, keepalive_backoff, max_clientid_len,
+                      mountpoint, is_bridge, connected_at}).
-                      session, stats_data, mountpoint, ws_initial_headers,
-                      peercert_username, is_bridge, connected_at}).
 
--type(proto_state() :: #proto_state{}).
+-define(INFO_KEYS, [capabilities, connected, client_id, clean_start, username, proto_ver, proto_name,
-
+                    keepalive, will_msg, mountpoint, is_bridge, connected_at]).
--define(INFO_KEYS, [client_id, username, clean_start, proto_ver, proto_name,
-                    keepalive, will_msg, ws_initial_headers, mountpoint,
-                    peercert_username, connected_at]).
 
 -define(STATS_KEYS, [recv_pkt, recv_msg, send_pkt, send_msg]).
 
 -define(LOG(Level, Format, Args, State),
         emqx_logger:Level([{client, State#proto_state.client_id}], "Client(~s@~s): " ++ Format,
-                              [State#proto_state.client_id, esockd_net:format(State#proto_state.peername) | Args])).
+                          [State#proto_state.client_id,
+                           esockd_net:format(maps:get(peername, State#proto_state.sockprops)) | Args])).
 
-%% @doc Init protocol
+-type(proto_state() :: #proto_state{}).
-init(Peername, SendFun, Opts) ->
+
-    Backoff = get_value(keepalive_backoff, Opts, 0.75),
+-export_type([proto_state/0]).
-    EnableStats = get_value(client_enable_stats, Opts, false),
+
-    MaxLen = get_value(max_clientid_len, Opts, ?MAX_CLIENTID_LEN),
+init(SockProps = #{zone := Zone, peercert := Peercert}, Options) ->
-    WsInitialHeaders = get_value(ws_initial_headers, Opts),
+    MountPoint = emqx_zone:get_env(Zone, mountpoint),
-    #proto_state{peername           = Peername,
+    Backoff = emqx_zone:get_env(Zone, keepalive_backoff, 0.75),
-                 sendfun            = SendFun,
+    Username = case proplists:get_value(peer_cert_as_username, Options) of
-                 max_clientid_len   = MaxLen,
+                   cn -> esockd_peercert:common_name(Peercert);
-                 is_superuser       = false,
+                   dn -> esockd_peercert:subject(Peercert);
+                   _  -> undefined
+               end,
+    #proto_state{sockprops          = SockProps,
+                 capabilities       = capabilities(Zone),
+                 connected          = false,
+                 clean_start        = true,
                  client_pid         = self(),
-                 peercert_username  = undefined,
+                 proto_ver          = ?MQTT_PROTO_V4,
-                 ws_initial_headers = WsInitialHeaders,
+                 proto_name         = <<"MQTT">>,
+                 username           = Username,
+                 is_superuser       = false,
                  keepalive_backoff  = Backoff,
-                 stats_data         = #proto_stats{enable_stats = EnableStats}}.
+                 mountpoint         = MountPoint,
+                 is_bridge          = false,
+                 recv_pkt           = 0,
+                 recv_msg           = 0,
+                 send_pkt           = 0,
+                 send_msg           = 0}.
 
-init(_Transport, _Sock, Peername, SendFun, Options) ->
+capabilities(Zone) ->
-    enrich_opt(Options, init(Peername, SendFun, Options)).
+    Capabilities = emqx_zone:get_env(Zone, mqtt_capabilities, []),
+    maps:from_list(lists:ukeymerge(1, ?CAPABILITIES, Capabilities)).
 
-enrich_opt([], State) ->
+parser(#proto_state{capabilities = #{max_packet_size := Size}, proto_ver = Ver}) ->
-    State;
+    emqx_frame:initial_state(#{max_packet_size => Size, version => Ver}).
-enrich_opt([{mountpoint, MountPoint} | ConnOpts], State) ->
-    enrich_opt(ConnOpts, State#proto_state{mountpoint = MountPoint});
-%%enrich_opt([{peer_cert_as_username, N} | ConnOpts], State) ->
-%%    enrich_opt(ConnOpts, State#proto_state{peercert_username = peercert_username(N, Conn)});
-enrich_opt([_ | ConnOpts], State) ->
-    enrich_opt(ConnOpts, State).
-
-%%peercert_username(cn, Conn) ->
-%%    Conn:peer_cert_common_name();
-%%peercert_username(dn, Conn) ->
-%%    Conn:peer_cert_subject().
-
-repl_username_with_peercert(State = #proto_state{peercert_username = undefined}) ->
-    State;
-repl_username_with_peercert(State = #proto_state{peercert_username = PeerCert}) ->
-    State#proto_state{username = PeerCert}.
-
-%%TODO::
-get(proto_ver, #proto_state{proto_ver = Ver}) ->
-    Ver;
-get(_, _ProtoState) ->
-    undefined.
 
 info(ProtoState) ->
     ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS).
 
-stats(#proto_state{stats_data = Stats}) ->
+stats(ProtoState) ->
-    tl(?record_to_proplist(proto_stats, Stats)).
+    ?record_to_proplist(proto_state, ProtoState, ?STATS_KEYS).
 
 clientid(#proto_state{client_id = ClientId}) ->
     ClientId.
 
-client(#proto_state{client_id          = ClientId,
+client(#proto_state{sockprops = #{peername := Peername},
-                    client_pid         = ClientPid,
+                    client_id = ClientId, client_pid = ClientPid, username  = Username}) ->
-                    peername           = Peername,
-                    username           = Username,
-                    clean_start        = CleanStart,
-                    proto_ver          = ProtoVer,
-                    keepalive          = Keepalive,
-                    will_msg           = WillMsg,
-                    ws_initial_headers = _WsInitialHeaders,
-                    mountpoint         = _MountPoint,
-                    connected_at       = _Time}) ->
-    WillTopic = if
-                    WillMsg =:= undefined -> undefined;
-                    true -> WillMsg#message.topic
-                end,
     #client{id = ClientId, pid = ClientPid, username = Username, peername = Peername}.
 
 session(#proto_state{session = Session}) ->
@@ -129,117 +107,93 @@ session(#proto_state{session = Session}) ->
 
 %% A Client can only send the CONNECT Packet once over a Network Connection.
 -spec(received(mqtt_packet(), proto_state()) -> {ok, proto_state()} | {error, term()}).
-received(Packet = ?PACKET(?CONNECT),
+received(Packet = ?PACKET(?CONNECT), ProtoState = #proto_state{connected = false}) ->
-         State = #proto_state{connected = false, stats_data = Stats}) ->
+    trace(recv, Packet, ProtoState),
-    trace(recv, Packet, State), Stats1 = inc_stats(recv, ?CONNECT, Stats),
+    process(Packet, inc_stats(recv, ?CONNECT, ProtoState#proto_state{connected = true}));
-    process(Packet, State#proto_state{connected = true, stats_data = Stats1});
 
 received(?PACKET(?CONNECT), State = #proto_state{connected = true}) ->
     {error, protocol_bad_connect, State};
 
 %% Received other packets when CONNECT not arrived.
-received(_Packet, State = #proto_state{connected = false}) ->
+received(_Packet, ProtoState = #proto_state{connected = false}) ->
-    {error, protocol_not_connected, State};
+    {error, protocol_not_connected, ProtoState};
 
-received(Packet = ?PACKET(Type), State = #proto_state{stats_data = Stats}) ->
+received(Packet = ?PACKET(Type), ProtoState) ->
-    trace(recv, Packet, State), Stats1 = inc_stats(recv, Type, Stats),
+    trace(recv, Packet, ProtoState),
     case validate_packet(Packet) of
         ok ->
-            process(Packet, State#proto_state{stats_data = Stats1});
+            process(Packet, inc_stats(recv, Type, ProtoState));
         {error, Reason} ->
-            {error, Reason, State}
+            {error, Reason, ProtoState}
     end.
 
-subscribe(RawTopicTable, ProtoState = #proto_state{client_id = ClientId,
+process(?CONNECT_PACKET(Var), ProtoState = #proto_state{username = Username0, client_pid = ClientPid}) ->
-                                                   username  = Username,
+    #mqtt_packet_connect{proto_name  = ProtoName,
-                                                   session   = Session}) ->
+                         proto_ver   = ProtoVer,
-    TopicTable = parse_topic_table(RawTopicTable),
-    case emqx_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of
-        {ok, TopicTable1} ->
-            emqx_session:subscribe(Session, TopicTable1);
-        {stop, _} ->
-            ok
-    end,
-    {ok, ProtoState}.
-
-unsubscribe(RawTopics, ProtoState = #proto_state{client_id = ClientId,
-                                                 username  = Username,
-                                                 session   = Session}) ->
-    case emqx_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
-        {ok, TopicTable} ->
-            emqx_session:unsubscribe(Session, TopicTable);
-        {stop, _} ->
-            ok
-    end,
-    {ok, ProtoState}.
-
-%% @doc Send PUBREL
-pubrel(PacketId, State) -> send(?PUBREL_PACKET(PacketId), State).
-
-process(?CONNECT_PACKET(Var), State0) ->
-
-    #mqtt_packet_connect{proto_ver  = ProtoVer,
-                         proto_name = ProtoName,
-                         username   = Username,
-                         password   = Password,
-                         clean_start= CleanStart,
-                         keepalive  = KeepAlive,
-                         client_id  = ClientId,
-                         is_bridge  = IsBridge} = Var,
-
-    State1 = repl_username_with_peercert(
-               State0#proto_state{proto_ver    = ProtoVer,
-                                  proto_name   = ProtoName,
-                                  username     = Username,
-                                  client_id    = ClientId,
-                                  clean_start  = CleanStart,
-                                  keepalive    = KeepAlive,
-                                  will_msg     = willmsg(Var, State0),
                          is_bridge   = IsBridge,
-                                  connected_at = os:timestamp()}),
+                         clean_start = CleanStart,
+                         keepalive   = Keepalive,
+                         properties  = ConnProps,
+                         client_id   = ClientId,
+                         username    = Username,
+                         password    = Password} = Var,
+    ProtoState1 = ProtoState#proto_state{proto_ver    = ProtoVer,
+                                         proto_name   = ProtoName,
+                                         username     = if Username0 == undefined ->
+                                                               Username;
+                                                           true -> Username0
+                                                        end, %% TODO: fixme later.
+                                         client_id    = ClientId,
+                                         clean_start  = CleanStart,
+                                         keepalive    = Keepalive,
+                                         connprops    = ConnProps,
+                                         will_msg     = willmsg(Var, ProtoState),
+                                         is_bridge    = IsBridge,
+                                         connected_at = os:timestamp()},
 
-    {ReturnCode1, SessPresent, State3} =
+    {ReturnCode1, SessPresent, ProtoState3} =
-    case validate_connect(Var, State1) of
+    case validate_connect(Var, ProtoState1) of
         ?RC_SUCCESS ->
-            case authenticate(client(State1), Password) of
+            case authenticate(client(ProtoState1), Password) of
                 {ok, IsSuperuser} ->
                     %% Generate clientId if null
-                    State2 = maybe_set_clientid(State1),
+                    ProtoState2 = maybe_set_clientid(ProtoState1),
-
+                    %% Open session
-                    %% Start session
                     case emqx_sm:open_session(#{clean_start => CleanStart,
-                                                client_id   => clientid(State2),
+                                                client_id   => clientid(ProtoState2),
                                                 username    => Username,
-                                                client_pid  => self()}) of
+                                                client_pid  => ClientPid}) of
                         {ok, Session} -> %% TODO:...
                             SP = true, %% TODO:...
                             %% TODO: Register the client
-                            emqx_cm:register_client(clientid(State2)),
+                            emqx_cm:register_client(clientid(ProtoState2)),
                             %%emqx_cm:reg(client(State2)),
                             %% Start keepalive
-                            start_keepalive(KeepAlive, State2),
+                            start_keepalive(Keepalive, ProtoState2),
                             %% Emit Stats
-                            self() ! emit_stats,
+                            %% self() ! emit_stats,
                             %% ACCEPT
-                            {?RC_SUCCESS, SP, State2#proto_state{session = Session, is_superuser = IsSuperuser}};
+                            {?RC_SUCCESS, SP, ProtoState2#proto_state{session = Session, is_superuser = IsSuperuser}};
                         {error, Error} ->
-                            {stop, {shutdown, Error}, State2}
+                            ?LOG(error, "Failed to open session: ~p", [Error], ProtoState2),
+                            {?RC_UNSPECIFIED_ERROR, false, ProtoState2} %% TODO: the error reason???
                     end;
                 {error, Reason}->
-                    ?LOG(error, "Username '~s' login failed for ~p", [Username, Reason], State1),
+                    ?LOG(error, "Username '~s' login failed for ~p", [Username, Reason], ProtoState1),
-                    {?RC_BAD_USER_NAME_OR_PASSWORD, false, State1}
+                    {?RC_BAD_USER_NAME_OR_PASSWORD, false, ProtoState1}
             end;
         ReturnCode ->
-            {ReturnCode, false, State1}
+            {ReturnCode, false, ProtoState1}
     end,
     %% Run hooks
-    emqx_hooks:run('client.connected', [ReturnCode1], client(State3)),
+    emqx_hooks:run('client.connected', [ReturnCode1], client(ProtoState3)),
     %%TODO: Send Connack
-    send(?CONNACK_PACKET(ReturnCode1, sp(SessPresent)), State3),
+    send(?CONNACK_PACKET(ReturnCode1, sp(SessPresent)), ProtoState3),
     %% stop if authentication failure
-    stop_if_auth_failure(ReturnCode1, State3);
+    stop_if_auth_failure(ReturnCode1, ProtoState3);
 
-process(Packet = ?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload), State = #proto_state{is_superuser = IsSuper}) ->
+process(Packet = ?PUBLISH_PACKET(_QoS, Topic, _PacketId, _Payload),
+        State = #proto_state{is_superuser = IsSuper}) ->
     case IsSuper orelse allow == check_acl(publish, Topic, client(State)) of
         true  -> publish(Packet, State);
         false -> ?LOG(error, "Cannot publish to ~s for ACL Deny", [Topic], State)
@@ -266,36 +220,49 @@ process(?SUBSCRIBE_PACKET(PacketId, []), State) ->
     send(?SUBACK_PACKET(PacketId, []), State);
 
 %% TODO: refactor later...
-process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable),
+process(?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), State) ->
-        State = #proto_state{client_id    = ClientId,
+    #proto_state{client_id    = ClientId,
                  username     = Username,
                  is_superuser = IsSuperuser,
                  mountpoint   = MountPoint,
-                             session      = Session}) ->
+                 session      = Session} = State,
-    Client = client(State), TopicTable = parse_topic_table(RawTopicTable),
+    Client = client(State),
+    TopicFilters = parse_topic_filters(RawTopicFilters),
     AllowDenies = if
                     IsSuperuser -> [];
-                    true -> [check_acl(subscribe, Topic, Client) || {Topic, _Opts} <- TopicTable]
+                    true -> [check_acl(subscribe, Topic, Client) || {Topic, _Opts} <- TopicFilters]
                   end,
     case lists:member(deny, AllowDenies) of
         true ->
-            ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicTable], State),
+            ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicFilters], State),
-            send(?SUBACK_PACKET(PacketId, [16#80 || _ <- TopicTable]), State);
+            send(?SUBACK_PACKET(PacketId, [?RC_NOT_AUTHORIZED || _ <- TopicFilters]), State);
         false ->
-            case emqx_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of
+            case emqx_hooks:run('client.subscribe', [ClientId, Username], TopicFilters) of
-                {ok, TopicTable1} ->
+                {ok, TopicFilters1} ->
-                    emqx_session:subscribe(Session, PacketId, mount(replvar(MountPoint, State), TopicTable1)),
+                    ok = emqx_session:subscribe(Session, {PacketId, Properties, mount(replvar(MountPoint, State), TopicFilters1)}),
                     {ok, State};
                 {stop, _} ->
                     {ok, State}
             end
     end;
 
+process({subscribe, RawTopicTable},
+        State = #proto_state{client_id = ClientId,
+                             username  = Username,
+                             session   = Session}) ->
+    TopicTable = parse_topic_filters(RawTopicTable),
+    case emqx_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of
+        {ok, TopicTable1} ->
+            emqx_session:subscribe(Session, TopicTable1);
+        {stop, _} -> ok
+    end,
+    {ok, State};
+
 %% Protect from empty topic list
 process(?UNSUBSCRIBE_PACKET(PacketId, []), State) ->
     send(?UNSUBACK_PACKET(PacketId), State);
 
-process(?UNSUBSCRIBE_PACKET(PacketId, RawTopics),
+process(?UNSUBSCRIBE_PACKET(PacketId, _Properties, RawTopics),
         State = #proto_state{client_id  = ClientId,
                              username   = Username,
                              mountpoint = MountPoint,
@@ -308,84 +275,106 @@ process(?UNSUBSCRIBE_PACKET(PacketId, RawTopics),
     end,
     send(?UNSUBACK_PACKET(PacketId), State);
 
-process(?PACKET(?PINGREQ), State) ->
+process({unsubscribe, RawTopics}, State = #proto_state{client_id = ClientId,
-    send(?PACKET(?PINGRESP), State);
+                                                       username  = Username,
+                                                       session   = Session}) ->
+    case emqx_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
+        {ok, TopicTable} ->
+            emqx_session:unsubscribe(Session, TopicTable);
+        {stop, _} -> ok
+    end,
+    {ok, State};
 
-process(?PACKET(?DISCONNECT), State) ->
+process(?PACKET(?PINGREQ), ProtoState) ->
+    send(?PACKET(?PINGRESP), ProtoState);
+
+process(?PACKET(?DISCONNECT), ProtoState) ->
     % Clean willmsg
-    {stop, normal, State#proto_state{will_msg = undefined}}.
+    {stop, normal, ProtoState#proto_state{will_msg = undefined}}.
 
-publish(Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId),
+deliver({publish, PacketId, Msg},
+        State = #proto_state{client_id  = ClientId,
+                             username   = Username,
+                             mountpoint = MountPoint,
+                             is_bridge  = IsBridge}) ->
+    emqx_hooks:run('message.delivered', [ClientId],
+                   emqx_message:set_header(username, Username, Msg)),
+    Msg1 = unmount(MountPoint, clean_retain(IsBridge, Msg)),
+    send(emqx_packet:from_message(PacketId, Msg1), State);
+
+deliver({pubrel, PacketId}, State) ->
+    send(?PUBREL_PACKET(PacketId), State);
+
+deliver({suback, PacketId, ReasonCodes}, ProtoState) ->
+    send(?SUBACK_PACKET(PacketId, ReasonCodes), ProtoState);
+
+deliver({unsuback, PacketId, ReasonCodes}, ProtoState) ->
+    send(?UNSUBACK_PACKET(PacketId, ReasonCodes), ProtoState).
+
+publish(Packet = ?PUBLISH_PACKET(?QOS_0, PacketId),
         State = #proto_state{client_id  = ClientId,
                              username   = Username,
                              mountpoint = MountPoint,
                              session    = Session}) ->
-    Msg = emqx_packet:to_message(Packet),
+    Msg = emqx_message:set_header(username, Username,
-    Msg1 = Msg#message{from = #client{id = ClientId, username = Username}},
+                                  emqx_packet:to_message(ClientId, Packet)),
-    emqx_session:publish(Session, mount(replvar(MountPoint, State), Msg1));
+    emqx_session:publish(Session, PacketId, mount(replvar(MountPoint, State), Msg));
 
-publish(Packet = ?PUBLISH_PACKET(?QOS_1, _PacketId), State) ->
+publish(Packet = ?PUBLISH_PACKET(?QOS_1), State) ->
     with_puback(?PUBACK, Packet, State);
 
-publish(Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) ->
+publish(Packet = ?PUBLISH_PACKET(?QOS_2), State) ->
     with_puback(?PUBREC, Packet, State).
 
-with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId),
+with_puback(Type, Packet = ?PUBLISH_PACKET(_QoS, PacketId),
             State = #proto_state{client_id  = ClientId,
                                  username   = Username,
                                  mountpoint = MountPoint,
                                  session    = Session}) ->
-    %% TODO: ...
+    Msg = emqx_message:set_header(username, Username,
-    Msg = emqx_packet:to_message(Packet),
+                                  emqx_packet:to_message(ClientId, Packet)),
-    Msg1 = Msg#message{from = #client{id = ClientId, username = Username}},
+    case emqx_session:publish(Session, PacketId, mount(replvar(MountPoint, State), Msg)) of
-    case emqx_session:publish(Session, mount(replvar(MountPoint, State), Msg1)) of
-        ok ->
-            case Type of
-                ?PUBACK -> send(?PUBACK_PACKET(PacketId), State);
-                ?PUBREC -> send(?PUBREC_PACKET(PacketId), State)
-            end;
         {error, Error} ->
-            ?LOG(error, "PUBLISH ~p error: ~p", [PacketId, Error], State)
+            ?LOG(error, "PUBLISH ~p error: ~p", [PacketId, Error], State);
+        _Delivery -> send({Type, PacketId}, State) %% TODO:
     end.
 
--spec(send(message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}).
+-spec(send({mqtt_packet_type(), mqtt_packet_id()} |
-send(Msg, State = #proto_state{client_id  = ClientId,
+           {mqtt_packet_id(), message()} |
-                               username   = Username,
+           mqtt_packet(), proto_state()) -> {ok, proto_state()}).
-                               mountpoint = MountPoint,
+send({?PUBACK, PacketId}, State) ->
-                               is_bridge  = IsBridge})
+    send(?PUBACK_PACKET(PacketId), State);
-        when is_record(Msg, message) ->
-    emqx_hooks:run('message.delivered', [ClientId, Username], Msg),
-    send(emqx_packet:from_message(unmount(MountPoint, clean_retain(IsBridge, Msg))), State);
 
-send(Packet = ?PACKET(Type), State = #proto_state{sendfun = SendFun, stats_data = Stats}) ->
+send({?PUBREC, PacketId}, State) ->
-    trace(send, Packet, State),
+    send(?PUBREC_PACKET(PacketId), State);
-    emqx_metrics:sent(Packet),
+
-    SendFun(Packet),
+send(Packet = ?PACKET(Type), ProtoState = #proto_state{proto_ver = Ver,
-    {ok, State#proto_state{stats_data = inc_stats(send, Type, Stats)}}.
+                                                       sockprops = #{sendfun := SendFun}}) ->
+    Data = emqx_frame:serialize(Packet, #{version => Ver}),
+    case SendFun(Data) of
+        ok -> emqx_metrics:sent(Packet),
+              trace(send, Packet, ProtoState),
+              {ok, inc_stats(send, Type, ProtoState)};
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 trace(recv, Packet, ProtoState) ->
-    ?LOG(info, "RECV ~s", [emqx_packet:format(Packet)], ProtoState);
+    ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)], ProtoState);
 
 trace(send, Packet, ProtoState) ->
-    ?LOG(info, "SEND ~s", [emqx_packet:format(Packet)], ProtoState).
+    ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)], ProtoState).
 
-inc_stats(_Direct, _Type, Stats = #proto_stats{enable_stats = false}) ->
+inc_stats(recv, Type, ProtoState = #proto_state{recv_pkt = PktCnt, recv_msg = MsgCnt}) ->
-    Stats;
+    ProtoState#proto_state{recv_pkt = PktCnt + 1,
-
+                           recv_msg = if Type =:= ?PUBLISH -> MsgCnt + 1;
-inc_stats(recv, Type, Stats) ->
+                                         true -> MsgCnt
-    #proto_stats{recv_pkt = PktCnt, recv_msg = MsgCnt} = Stats,
+                                      end};
-    inc_stats(Type, #proto_stats.recv_pkt, PktCnt, #proto_stats.recv_msg, MsgCnt, Stats);
+inc_stats(send, Type, ProtoState = #proto_state{send_pkt = PktCnt, send_msg = MsgCnt}) ->
-
+    ProtoState#proto_state{send_pkt = PktCnt + 1,
-inc_stats(send, Type, Stats) ->
+                           send_msg = if Type =:= ?PUBLISH -> MsgCnt + 1;
-    #proto_stats{send_pkt = PktCnt, send_msg = MsgCnt} = Stats,
+                                         true -> MsgCnt
-    inc_stats(Type, #proto_stats.send_pkt, PktCnt, #proto_stats.send_msg, MsgCnt, Stats).
+                                      end}.
-
-inc_stats(Type, PktPos, PktCnt, MsgPos, MsgCnt, Stats) ->
-    Stats1 = setelement(PktPos, Stats, PktCnt + 1),
-    case Type =:= ?PUBLISH of
-        true  -> setelement(MsgPos, Stats1, MsgCnt + 1);
-        false -> Stats1
-    end.
 
 stop_if_auth_failure(?RC_SUCCESS, State) ->
     {ok, State};
@@ -403,19 +392,18 @@ shutdown(mnesia_conflict, _State = #proto_state{client_id = ClientId}) ->
 shutdown(Error, State = #proto_state{client_id = ClientId,
                                      will_msg  = WillMsg}) ->
     ?LOG(info, "Shutdown for ~p", [Error], State),
-    Client = client(State),
     %% Auth failure not publish the will message
     case Error =:= auth_failure of
         true -> ok;
-        false -> send_willmsg(Client, WillMsg)
+        false -> send_willmsg(ClientId, WillMsg)
     end,
-    emqx_hooks:run('client.disconnected', [Error], Client),
+    emqx_hooks:run('client.disconnected', [Error], client(State)),
     emqx_cm:unregister_client(ClientId),
     ok.
 
-willmsg(Packet, State = #proto_state{mountpoint = MountPoint})
+willmsg(Packet, State = #proto_state{client_id = ClientId, mountpoint = MountPoint})
     when is_record(Packet, mqtt_packet_connect) ->
-    case emqx_packet:to_message(Packet) of
+    case emqx_packet:to_message(ClientId, Packet) of
         undefined -> undefined;
         Msg -> mount(replvar(MountPoint, State), Msg)
     end.
@@ -430,10 +418,10 @@ maybe_set_clientid(State = #proto_state{client_id = NullId})
 maybe_set_clientid(State) ->
     State.
 
-send_willmsg(_Client, undefined) ->
+send_willmsg(_ClientId, undefined) ->
     ignore;
-send_willmsg(Client, WillMsg) ->
+send_willmsg(ClientId, WillMsg) ->
-    emqx_broker:publish(WillMsg#message{from = Client}).
+    emqx_broker:publish(WillMsg#message{from = ClientId}).
 
 start_keepalive(0, _State) -> ignore;
 
@@ -459,7 +447,7 @@ validate_protocol(#mqtt_packet_connect{proto_ver = Ver, proto_name = Name}) ->
     lists:member({Ver, Name}, ?PROTOCOL_NAMES).
 
 validate_clientid(#mqtt_packet_connect{client_id = ClientId},
-                  #proto_state{max_clientid_len = MaxLen})
+                  #proto_state{capabilities = #{max_clientid_len := MaxLen}})
     when (byte_size(ClientId) >= 1) andalso (byte_size(ClientId) =< MaxLen) ->
     true;
 
@@ -481,7 +469,7 @@ validate_clientid(#mqtt_packet_connect{proto_ver   = ProtoVer,
          [ProtoVer, CleanStart], ProtoState),
     false.
 
-validate_packet(?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload)) ->
+validate_packet(?PUBLISH_PACKET(_QoS, Topic, _PacketId, _Payload)) ->
     case emqx_topic:validate({name, Topic}) of
         true  -> ok;
         false -> {error, badtopic}
@@ -501,11 +489,11 @@ validate_topics(_Type, []) ->
 
 validate_topics(Type, TopicTable = [{_Topic, _SubOpts}|_])
     when Type =:= name orelse Type =:= filter ->
-    Valid = fun(Topic, Qos) ->
+    Valid = fun(Topic, QoS) ->
-              emqx_topic:validate({Type, Topic}) and validate_qos(Qos)
+              emqx_topic:validate({Type, Topic}) and validate_qos(QoS)
             end,
     case [Topic || {Topic, SubOpts} <- TopicTable,
-                   not Valid(Topic, proplists:get_value(qos, SubOpts))] of
+                   not Valid(Topic, SubOpts#mqtt_subopts.qos)] of
         [] -> ok;
         _  -> {error, badtopic}
     end;
@@ -518,17 +506,16 @@ validate_topics(Type, Topics = [Topic0|_]) when is_binary(Topic0) ->
 
 validate_qos(undefined) ->
     true;
-validate_qos(Qos) when ?IS_QOS(Qos) ->
+validate_qos(QoS) when ?IS_QOS(QoS) ->
     true;
 validate_qos(_) ->
     false.
 
-parse_topic_table(TopicTable) ->
+parse_topic_filters(TopicFilters) ->
-    lists:map(fun({Topic0, SubOpts}) ->
+    [begin
-                {Topic, Opts} = emqx_topic:parse(Topic0),
+         {Topic, Opts} = emqx_topic:parse(RawTopic),
-                %%TODO:
+         {Topic, maps:merge(?record_to_map(mqtt_subopts, SubOpts), Opts)}
-                {Topic, lists:usort(lists:umerge(Opts, SubOpts))}
+     end || {RawTopic, SubOpts} <- TopicFilters].
-        end, TopicTable).
 
 parse_topics(Topics) ->
     [emqx_topic:parse(Topic) || Topic <- Topics].

src/emqx_router.erl

@@ -33,10 +33,8 @@
 -export([del_route/1, del_route/2, del_route/3]).
 -export([has_routes/1, match_routes/1, print_routes/1]).
 -export([topics/0]).
-
+%% gen_server callbacks
-%% gen_server Function Exports
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
-         code_change/3]).
 
 -type(destination() :: node() | {binary(), node()}).
 

src/emqx_session.erl

@@ -11,29 +11,30 @@
 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
-
+%%
+%% @doc
 %% A stateful interaction between a Client and a Server. Some Sessions
 %% last only as long as the Network Connection, others can span multiple
 %% consecutive Network Connections between a Client and a Server.
 %%
-%% The Session state in the Server consists of:
+%% The Session State in the Server consists of:
 %%
-%% The existence of a Session, even if the rest of the Session state is empty.
+%% The existence of a Session, even if the rest of the Session State is empty.
 %%
-%% The Client’s subscriptions.
+%% The Clients subscriptions, including any Subscription Identifiers.
 %%
 %% QoS 1 and QoS 2 messages which have been sent to the Client, but have not
 %% been completely acknowledged.
 %%
-%% QoS 1 and QoS 2 messages pending transmission to the Client.
+%% QoS 1 and QoS 2 messages pending transmission to the Client and OPTIONALLY
+%% QoS 0 messages pending transmission to the Client.
 %%
-%% QoS 2 messages which have been received from the Client, but have not
+%% QoS 2 messages which have been received from the Client, but have not been
-%% been completely acknowledged.
+%% completely acknowledged.The Will Message and the Will Delay Interval
 %%
-%% Optionally, QoS 0 messages pending transmission to the Client.
+%% If the Session is currently not connected, the time at which the Session
-%%
+%% will end and Session State will be discarded.
-%% If the session is currently disconnected, the time at which the Session state
+%% @end
-%% will be deleted.
 -module(emqx_session).
 
 -behaviour(gen_server).
@@ -42,26 +43,20 @@
 -include("emqx_mqtt.hrl").
 -include("emqx_misc.hrl").
 
--import(emqx_misc, [start_timer/2]).
+-export([start_link/1, close/1]).
--import(proplists, [get_value/2, get_value/3]).
+-export([info/1, stats/1]).
-
+-export([resume/2, discard/2]).
-%% Session API
+-export([subscribe/2]).%%, subscribe/3]).
--export([start_link/1, resume/2, discard/2]).
+-export([publish/3]).
-%% Management and Monitor API
+-export([puback/2, pubrec/2, pubrel/2, pubcomp/2]).
--export([state/1, info/1, stats/1]).
-%% PubSub API
--export([subscribe/2, subscribe/3]).
--export([publish/2, puback/2, pubrec/2, pubrel/2, pubcomp/2]).
 -export([unsubscribe/2]).
 
-%% gen_server Function Exports
+%% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 
--define(MQueue, emqx_mqueue).
+-record(state, {
-
+          %% Clean Start Flag
--record(state,
-        { %% Clean Start Flag
           clean_start = false :: boolean(),
 
           %% Client Binding: local | remote
@@ -73,21 +68,25 @@
           %% Username
           username :: binary() | undefined,
 
-          %% Client Pid binding with session
+          %% Client pid binding with session
           client_pid :: pid(),
 
-          %% Old Client Pid that has been kickout
+          %% Old client Pid that has been kickout
           old_client_pid :: pid(),
 
-          %% Next message id of the session
+          %% Pending sub/unsub requests
-          next_msg_id = 1 :: mqtt_packet_id(),
+          requests :: map(),
 
+          %% Next packet id of the session
+          next_pkt_id = 1 :: mqtt_packet_id(),
+
+          %% Max subscriptions
           max_subscriptions :: non_neg_integer(),
 
-          %% Client’s subscriptions.
+          %% Client’s Subscriptions.
           subscriptions :: map(),
 
-          %% Upgrade Qos?
+          %% Upgrade QoS?
           upgrade_qos = false :: boolean(),
 
           %% Client <- Broker: Inflight QoS1, QoS2 messages sent to the client but unacked.
@@ -106,18 +105,18 @@
           %% QoS 1 and QoS 2 messages pending transmission to the Client.
           %%
           %% Optionally, QoS 0 messages pending transmission to the Client.
-          mqueue :: ?MQueue:mqueue(),
+          mqueue :: emqx_mqueue:mqueue(),
 
           %% Client -> Broker: Inflight QoS2 messages received from client and waiting for pubrel.
           awaiting_rel :: map(),
 
-          %% Max Packets that Awaiting PUBREL
+          %% Max Packets Awaiting PUBREL
           max_awaiting_rel = 100 :: non_neg_integer(),
 
-          %% Awaiting PUBREL timeout
+          %% Awaiting PUBREL Timeout
           await_rel_timeout = 20000 :: timeout(),
 
-          %% Awaiting PUBREL timer
+          %% Awaiting PUBREL Timer
           await_rel_timer :: reference() | undefined,
 
           %% Session Expiry Interval
@@ -135,13 +134,14 @@
           %% Ignore loop deliver?
           ignore_loop_deliver = false :: boolean(),
 
+          %% Created at
           created_at :: erlang:timestamp()
         }).
 
 -define(TIMEOUT, 60000).
 -define(INFO_KEYS, [clean_start, client_id, username, client_pid, binding, created_at]).
 -define(STATE_KEYS, [clean_start, client_id, username, binding, client_pid, old_client_pid,
-                     next_msg_id, max_subscriptions, subscriptions, upgrade_qos, inflight,
+                     next_pkt_id, max_subscriptions, subscriptions, upgrade_qos, inflight,
                      max_inflight, retry_interval, mqueue, awaiting_rel, max_awaiting_rel,
                      await_rel_timeout, expiry_interval, enable_stats, force_gc_count,
                      created_at]).
@@ -150,49 +150,47 @@
         emqx_logger:Level([{client, State#state.client_id}],
                           "Session(~s): " ++ Format, [State#state.client_id | Args])).
 
-%% @doc Start a Session
+%% @doc Start a session
--spec(start_link(map()) -> {ok, pid()} | {error, term()}).
+-spec(start_link(Attrs :: map()) -> {ok, pid()} | {error, term()}).
 start_link(Attrs) ->
     gen_server:start_link(?MODULE, Attrs, [{hibernate_after, 10000}]).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% PubSub API
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
-%% @doc Subscribe topics
+%% for mqtt 5.0
--spec(subscribe(pid(), [{binary(), [emqx_topic:option()]}]) -> ok).
+-spec(subscribe(pid(), {mqtt_packet_id(), mqtt_properties(), topic_table()}) -> ok).
-subscribe(SPid, TopicTable) -> %%TODO: the ack function??...
+subscribe(SPid, SubReq = {PacketId, Props, TopicFilters}) ->
-    gen_server:cast(SPid, {subscribe, self(), TopicTable, fun(_) -> ok end}).
+    gen_server:cast(SPid, {subscribe, self(), SubReq}).
 
--spec(subscribe(pid(), mqtt_packet_id(), [{binary(), [emqx_topic:option()]}]) -> ok).
+-spec(publish(pid(), mqtt_packet_id(), message()) -> {ok, delivery()} | {error, term()}).
-subscribe(SPid, PacketId, TopicTable) -> %%TODO: the ack function??...
+publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) ->
-    From = self(),
+    %% Publish QoS0 message to broker directly
-    AckFun = fun(GrantedQos) -> From ! {suback, PacketId, GrantedQos} end,
-    gen_server:cast(SPid, {subscribe, From, TopicTable, AckFun}).
-
-%% @doc Publish Message
--spec(publish(pid(), message()) -> {ok, delivery()} | {error, term()}).
-publish(_SPid, Msg = #message{qos = ?QOS_0}) ->
-    %% Publish QoS0 Directly
     emqx_broker:publish(Msg);
 
-publish(_SPid, Msg = #message{qos = ?QOS_1}) ->
+publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) ->
-    %% Publish QoS1 message directly for client will PubAck automatically
+    %% Publish QoS1 message to broker directly
     emqx_broker:publish(Msg);
 
-publish(SPid, Msg = #message{qos = ?QOS_2}) ->
+publish(SPid, PacketId, Msg = #message{qos = ?QOS_2}) ->
-    %% Publish QoS2 to Session
+    %% Publish QoS2 message to session
-    gen_server:call(SPid, {publish, Msg}, infinity).
+    gen_server:call(SPid, {publish, PacketId, Msg}, infinity).
 
-%% @doc PubAck Message
 -spec(puback(pid(), mqtt_packet_id()) -> ok).
 puback(SPid, PacketId) ->
     gen_server:cast(SPid, {puback, PacketId}).
 
+puback(SPid, PacketId, {ReasonCode, Props}) ->
+    gen_server:cast(SPid, {puback, PacketId, {ReasonCode, Props}}).
+
 -spec(pubrec(pid(), mqtt_packet_id()) -> ok).
 pubrec(SPid, PacketId) ->
     gen_server:cast(SPid, {pubrec, PacketId}).
 
+pubrec(SPid, PacketId, {ReasonCode, Props}) ->
+    gen_server:cast(SPid, {pubrec, PacketId, {ReasonCode, Props}}).
+
 -spec(pubrel(pid(), mqtt_packet_id()) -> ok).
 pubrel(SPid, PacketId) ->
     gen_server:cast(SPid, {pubrel, PacketId}).
@@ -201,20 +199,14 @@ pubrel(SPid, PacketId) ->
 pubcomp(SPid, PacketId) ->
     gen_server:cast(SPid, {pubcomp, PacketId}).
 
-%% @doc Unsubscribe the topics
+-spec(unsubscribe(pid(), {mqtt_packet_id(), mqtt_properties(), topic_table()}) -> ok).
--spec(unsubscribe(pid(), [{binary(), [suboption()]}]) -> ok).
+unsubscribe(SPid, UnsubReq = {PacketId, Properties, TopicFilters}) ->
-unsubscribe(SPid, TopicTable) ->
+    gen_server:cast(SPid, {unsubscribe, self(), UnsubReq}).
-    gen_server:cast(SPid, {unsubscribe, self(), TopicTable}).
 
-%% @doc Resume the session
 -spec(resume(pid(), pid()) -> ok).
 resume(SPid, ClientPid) ->
     gen_server:cast(SPid, {resume, ClientPid}).
 
-%% @doc Get session state
-state(SPid) when is_pid(SPid) ->
-    gen_server:call(SPid, state).
-
 %% @doc Get session info
 -spec(info(pid() | #state{}) -> list(tuple())).
 info(SPid) when is_pid(SPid) ->
@@ -239,9 +231,9 @@ stats(#state{max_subscriptions = MaxSubscriptions,
                   {subscriptions,     maps:size(Subscriptions)},
                   {max_inflight,      MaxInflight},
                   {inflight_len,      emqx_inflight:size(Inflight)},
-                  {max_mqueue,        ?MQueue:max_len(MQueue)},
+                  {max_mqueue,        emqx_mqueue:max_len(MQueue)},
-                  {mqueue_len,        ?MQueue:len(MQueue)},
+                  {mqueue_len,        emqx_mqueue:len(MQueue)},
-                  {mqueue_dropped,    ?MQueue:dropped(MQueue)},
+                  {mqueue_dropped,    emqx_mqueue:dropped(MQueue)},
                   {max_awaiting_rel,  MaxAwaitingRel},
                   {awaiting_rel_len,  maps:size(AwaitingRel)},
                   {deliver_msg,       get(deliver_msg)},
@@ -250,41 +242,42 @@ stats(#state{max_subscriptions = MaxSubscriptions,
 %% @doc Discard the session
 -spec(discard(pid(), client_id()) -> ok).
 discard(SPid, ClientId) ->
-    gen_server:call(SPid, {discard, ClientId}).
+    gen_server:call(SPid, {discard, ClientId}, infinity).
 
-%%--------------------------------------------------------------------
+-spec(close(pid()) -> ok).
-%% gen_server Callbacks
+close(SPid) ->
-%%--------------------------------------------------------------------
+    gen_server:call(SPid, close, infinity).
 
-init(#{clean_start := CleanStart,
+%%------------------------------------------------------------------------------
-       client_id   := ClientId,
+%% gen_server callbacks
-       username    := Username,
+%%------------------------------------------------------------------------------
-       client_pid  := ClientPid}) ->
+
+init(#{clean_start := CleanStart, client_id := ClientId, username := Username, client_pid  := ClientPid}) ->
     process_flag(trap_exit, true),
     true = link(ClientPid),
     init_stats([deliver_msg, enqueue_msg]),
     {ok, Env} = emqx_config:get_env(session),
     {ok, QEnv} = emqx_config:get_env(mqueue),
-    MaxInflight = get_value(max_inflight, Env, 0),
+    MaxInflight = proplists:get_value(max_inflight, Env, 0),
-    EnableStats = get_value(enable_stats, Env, false),
+    EnableStats = proplists:get_value(enable_stats, Env, false),
-    IgnoreLoopDeliver = get_value(ignore_loop_deliver, Env, false),
+    IgnoreLoopDeliver = proplists:get_value(ignore_loop_deliver, Env, false),
-    MQueue = ?MQueue:new(ClientId, QEnv),
+    MQueue = emqx_mqueue:new(ClientId, QEnv),
     State = #state{clean_start       = CleanStart,
                    binding           = binding(ClientPid),
                    client_id         = ClientId,
                    client_pid        = ClientPid,
                    username          = Username,
                    subscriptions     = #{},
-                   max_subscriptions = get_value(max_subscriptions, Env, 0),
+                   max_subscriptions = proplists:get_value(max_subscriptions, Env, 0),
-                   upgrade_qos       = get_value(upgrade_qos, Env, false),
+                   upgrade_qos       = proplists:get_value(upgrade_qos, Env, false),
                    max_inflight      = MaxInflight,
                    inflight          = emqx_inflight:new(MaxInflight),
                    mqueue            = MQueue,
-                   retry_interval    = get_value(retry_interval, Env),
+                   retry_interval    = proplists:get_value(retry_interval, Env),
                    awaiting_rel      = #{},
-                   await_rel_timeout = get_value(await_rel_timeout, Env),
+                   await_rel_timeout = proplists:get_value(await_rel_timeout, Env),
-                   max_awaiting_rel  = get_value(max_awaiting_rel, Env),
+                   max_awaiting_rel  = proplists:get_value(max_awaiting_rel, Env),
-                   expiry_interval   = get_value(expiry_interval, Env),
+                   expiry_interval   = proplists:get_value(expiry_interval, Env),
                    enable_stats      = EnableStats,
                    ignore_loop_deliver = IgnoreLoopDeliver,
                    created_at        = os:timestamp()},
@@ -307,19 +300,19 @@ handle_call({discard, ClientPid}, _From, State = #state{client_pid = OldClientPi
     ?LOG(warning, " ~p kickout ~p", [ClientPid, OldClientPid], State),
     {stop, {shutdown, conflict}, ok, State};
 
-handle_call({publish, Msg = #message{qos = ?QOS_2, headers = #{packet_id := PacketId}}}, _From,
+handle_call({publish, PacketId, Msg = #message{qos = ?QOS_2}}, _From,
             State = #state{awaiting_rel      = AwaitingRel,
                            await_rel_timer   = Timer,
                            await_rel_timeout = Timeout}) ->
     case is_awaiting_full(State) of
         false ->
             State1 = case Timer == undefined of
-                         true  -> State#state{await_rel_timer = start_timer(Timeout, check_awaiting_rel)};
+                         true  -> State#state{await_rel_timer = emqx_misc:start_timer(Timeout, check_awaiting_rel)};
                          false -> State
                      end,
             reply(ok, State1#state{awaiting_rel = maps:put(PacketId, Msg, AwaitingRel)});
         true ->
-            ?LOG(warning, "Dropped Qos2 Message for too many awaiting_rel: ~p", [Msg], State),
+            ?LOG(warning, "Dropped QoS2 Message for too many awaiting_rel: ~p", [Msg], State),
             emqx_metrics:inc('messages/qos2/dropped'),
             reply({error, dropped}, State)
     end;
@@ -330,62 +323,53 @@ handle_call(info, _From, State) ->
 handle_call(stats, _From, State) ->
     reply(stats(State), State);
 
-handle_call(state, _From, State) ->
+handle_call(close, _From, State) ->
-    reply(?record_to_proplist(state, State, ?STATE_KEYS), State);
+    {stop, normal, State};
 
 handle_call(Req, _From, State) ->
     emqx_logger:error("[Session] unexpected call: ~p", [Req]),
     {reply, ignored, State}.
 
-handle_cast({subscribe, From, TopicTable, AckFun},
+handle_cast({subscribe, From, {PacketId, _Properties, TopicFilters}},
             State = #state{client_id = ClientId, username = Username, subscriptions = Subscriptions}) ->
-    ?LOG(info, "Subscribe ~p", [TopicTable], State),
+    ?LOG(info, "Subscribe ~p", [TopicFilters], State),
-    {GrantedQos, Subscriptions1} =
+    {ReasonCodes, Subscriptions1} =
-    lists:foldl(fun({Topic, Opts}, {QosAcc, SubMap}) ->
+    lists:foldl(fun({Topic, SubOpts = #{qos := QoS}}, {RcAcc, SubMap}) ->
-                NewQos = get_value(qos, Opts),
+                    {[QoS|RcAcc],
-                SubMap1 =
                      case maps:find(Topic, SubMap) of
-                    {ok, NewQos} ->
+                         {ok, SubOpts} ->
-                        ?LOG(warning, "Duplicated subscribe: ~s, qos = ~w", [Topic, NewQos], State),
+                             ?LOG(warning, "Duplicated subscribe: ~s, subopts: ~p", [Topic, SubOpts], State),
                              SubMap;
-                    {ok, OldQos} ->
+                         {ok, OldOpts} ->
-                        %% TODO:....
+                             emqx_broker:set_subopts(Topic, {self(), ClientId}, SubOpts),
-                        emqx_broker:set_subopts(Topic, ClientId, [{qos, NewQos}]),
+                             emqx_hooks:run('session.subscribed', [ClientId, Username], {Topic, SubOpts}),
-                        emqx_hooks:run('session.subscribed', [ClientId, Username], {Topic, Opts}),
+                             ?LOG(warning, "Duplicated subscribe ~s, old_opts: ~p, new_opts: ~p", [Topic, OldOpts, SubOpts], State),
-                        ?LOG(warning, "Duplicated subscribe ~s, old_qos=~w, new_qos=~w", [Topic, OldQos, NewQos], State),
+                             maps:put(Topic, SubOpts, SubMap);
-                        maps:put(Topic, NewQos, SubMap);
                          error ->
-                        %% TODO:....
+                             emqx_broker:subscribe(Topic, ClientId, SubOpts),
-                        emqx:subscribe(Topic, ClientId, Opts),
+                             emqx_hooks:run('session.subscribed', [ClientId, Username], {Topic, SubOpts}),
-                        emqx_hooks:run('session.subscribed', [ClientId, Username], {Topic, Opts}),
+                             maps:put(Topic, SubOpts, SubMap)
-                        maps:put(Topic, NewQos, SubMap)
+                     end}
-                end,
+                end, {[], Subscriptions}, TopicFilters),
-                {[NewQos|QosAcc], SubMap1}
+    suback(From, PacketId, lists:reverse(ReasonCodes)),
-        end, {[], Subscriptions}, TopicTable),
+    {noreply, emit_stats(State#state{subscriptions = Subscriptions1})};
-    AckFun(lists:reverse(GrantedQos)),
-    {noreply, emit_stats(State#state{subscriptions = Subscriptions1}), hibernate};
 
-handle_cast({unsubscribe, From, TopicTable},
+handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}},
-            State = #state{client_id     = ClientId,
+            State = #state{client_id = ClientId, username = Username, subscriptions = Subscriptions}) ->
-                           username      = Username,
+    ?LOG(info, "Unsubscribe ~p", [TopicFilters], State),
-                           subscriptions = Subscriptions}) ->
+    {ReasonCodes, Subscriptions1} =
-    ?LOG(info, "Unsubscribe ~p", [TopicTable], State),
+    lists:foldl(fun(Topic, {RcAcc, SubMap}) ->
-    Subscriptions1 =
-    lists:foldl(fun({Topic, Opts}, SubMap) ->
-                Fastlane = lists:member(fastlane, Opts),
                 case maps:find(Topic, SubMap) of
-                    {ok, _Qos} ->
+                    {ok, SubOpts} ->
-                        case Fastlane of
+                        emqx_broker:unsubscribe(Topic, ClientId),
-                            true  -> emqx:unsubscribe(Topic, From);
+                        emqx_hooks:run('session.unsubscribed', [ClientId, Username], {Topic, SubOpts}),
-                            false -> emqx:unsubscribe(Topic, ClientId)
+                        {[?RC_SUCCESS|RcAcc], maps:remove(Topic, SubMap)};
-                        end,
-                        emqx_hooks:run('session.unsubscribed', [ClientId, Username], {Topic, Opts}),
-                        maps:remove(Topic, SubMap);
                     error ->
-                        SubMap
+                        {[?RC_NO_SUBSCRIPTION_EXISTED|RcAcc], SubMap}
                 end
-        end, Subscriptions, TopicTable),
+        end, {[], Subscriptions}, TopicFilters),
-    {noreply, emit_stats(State#state{subscriptions = Subscriptions1}), hibernate};
+    unsuback(From, PacketId, lists:reverse(ReasonCodes)),
+    {noreply, emit_stats(State#state{subscriptions = Subscriptions1})};
 
 %% PUBACK:
 handle_cast({puback, PacketId}, State = #state{inflight = Inflight}) ->
@@ -490,12 +474,12 @@ handle_cast(Msg, State) ->
     {noreply, State}.
 
 %% Ignore Messages delivered by self
-handle_info({dispatch, _Topic, #message{from = {ClientId, _}}},
+handle_info({dispatch, _Topic, #message{from = ClientId}},
              State = #state{client_id = ClientId, ignore_loop_deliver = true}) ->
     {noreply, State};
 
 %% Dispatch Message
-handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, message) ->
+handle_info({dispatch, Topic, Msg}, State) ->
     {noreply, gc(dispatch(tune_qos(Topic, reset_dup(Msg), State), State))};
 
 %% Do nothing if the client has been disconnected.
@@ -521,7 +505,7 @@ handle_info({'EXIT', ClientPid, Reason},
                            client_pid      = ClientPid,
                            expiry_interval = Interval}) ->
     ?LOG(info, "Client ~p EXIT for ~p", [ClientPid, Reason], State),
-    ExpireTimer = start_timer(Interval, expired),
+    ExpireTimer = emqx_misc:start_timer(Interval, expired),
     State1 = State#state{client_pid = undefined, expiry_timer = ExpireTimer},
     {noreply, emit_stats(State1), hibernate};
 
@@ -543,12 +527,25 @@ terminate(Reason, #state{client_id = ClientId, username = Username}) ->
     emqx_hooks:run('session.terminated', [ClientId, Username, Reason]),
     emqx_sm:unregister_session(ClientId).
 
-code_change(_OldVsn, Session, _Extra) ->
+code_change(_OldVsn, State, _Extra) ->
-    {ok, Session}.
+    {ok, State}.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+suback(_From, undefined, _ReasonCodes) ->
+    ignore;
+suback(From, PacketId, ReasonCodes) ->
+    From ! {deliver, {suback, PacketId, ReasonCodes}}.
+
+unsuback(_From, undefined, _ReasonCodes) ->
+    ignore;
+unsuback(From, PacketId, ReasonCodes) ->
+    From ! {deliver, {unsuback, PacketId, ReasonCodes}}.
+
+%%------------------------------------------------------------------------------
 %% Kickout old client
-%%--------------------------------------------------------------------
 
 kick(_ClientId, undefined, _Pid) ->
     ignore;
@@ -560,32 +557,32 @@ kick(ClientId, OldPid, Pid) ->
     %% Clean noproc
     receive {'EXIT', OldPid, _} -> ok after 0 -> ok end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Replay or Retry Delivery
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
-%% Redeliver at once if Force is true
+%% Redeliver at once if force is true
 retry_delivery(Force, State = #state{inflight = Inflight}) ->
     case emqx_inflight:is_empty(Inflight) of
-        true  -> State;
+        true  ->
-        false -> Msgs = lists:sort(sortfun(inflight),
+            State;
-                                   emqx_inflight:values(Inflight)),
+        false ->
+            Msgs = lists:sort(sortfun(inflight), emqx_inflight:values(Inflight)),
             retry_delivery(Force, Msgs, os:timestamp(), State)
     end.
 
 retry_delivery(_Force, [], _Now, State = #state{retry_interval = Interval}) ->
-    State#state{retry_timer = start_timer(Interval, retry_delivery)};
+    State#state{retry_timer = emqx_misc:start_timer(Interval, retry_delivery)};
 
-retry_delivery(Force, [{Type, Msg, Ts} | Msgs], Now,
+retry_delivery(Force, [{Type, Msg0, Ts} | Msgs], Now,
-               State = #state{inflight       = Inflight,
+               State = #state{inflight = Inflight, retry_interval = Interval}) ->
-                              retry_interval = Interval}) ->
     Diff = timer:now_diff(Now, Ts) div 1000, %% micro -> ms
     if
         Force orelse (Diff >= Interval) ->
-            case {Type, Msg} of
+            case {Type, Msg0} of
-                {publish, Msg = #message{headers = #{packet_id := PacketId}}} ->
+                {publish, {PacketId, Msg}} ->
-                    redeliver(Msg, State),
+                    redeliver({PacketId, Msg}, State),
-                    Inflight1 = emqx_inflight:update(PacketId, {publish, Msg, Now}, Inflight),
+                    Inflight1 = emqx_inflight:update(PacketId, {publish, {PacketId, Msg}, Now}, Inflight),
                     retry_delivery(Force, Msgs, Now, State#state{inflight = Inflight1});
                 {pubrel, PacketId} ->
                     redeliver({pubrel, PacketId}, State),
@@ -593,12 +590,12 @@ retry_delivery(Force, [{Type, Msg, Ts} | Msgs], Now,
                     retry_delivery(Force, Msgs, Now, State#state{inflight = Inflight1})
             end;
         true ->
-            State#state{retry_timer = start_timer(Interval - Diff, retry_delivery)}
+            State#state{retry_timer = emqx_misc:start_timer(Interval - Diff, retry_delivery)}
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Expire Awaiting Rel
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 expire_awaiting_rel(State = #state{awaiting_rel = AwaitingRel}) ->
     case maps:size(AwaitingRel) of
@@ -619,12 +616,12 @@ expire_awaiting_rel([{PacketId, Msg = #message{timestamp = TS}} | Msgs],
             emqx_metrics:inc('messages/qos2/dropped'),
             expire_awaiting_rel(Msgs, Now, State#state{awaiting_rel = maps:remove(PacketId, AwaitingRel)});
         Diff ->
-            State#state{await_rel_timer = start_timer(Timeout - Diff, check_awaiting_rel)}
+            State#state{await_rel_timer = emqx_misc:start_timer(Timeout - Diff, check_awaiting_rel)}
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Sort Inflight, AwaitingRel
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 sortfun(inflight) ->
     fun({_, _, Ts1}, {_, _, Ts2}) -> Ts1 < Ts2 end;
@@ -635,18 +632,18 @@ sortfun(awaiting_rel) ->
         Ts1 < Ts2
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Check awaiting rel
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 is_awaiting_full(#state{max_awaiting_rel = 0}) ->
     false;
 is_awaiting_full(#state{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLen}) ->
     maps:size(AwaitingRel) >= MaxLen.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Dispatch Messages
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 %% Enqueue message if the client has been disconnected
 dispatch(Msg, State = #state{client_id = ClientId, client_pid = undefined}) ->
@@ -657,53 +654,50 @@ dispatch(Msg, State = #state{client_id = ClientId, client_pid = undefined}) ->
 
 %% Deliver qos0 message directly to client
 dispatch(Msg = #message{qos = ?QOS0}, State) ->
-    deliver(Msg, State), State;
+    deliver(undefined, Msg, State), State;
 
-dispatch(Msg = #message{qos = QoS},
+dispatch(Msg = #message{qos = QoS}, State = #state{next_pkt_id = PacketId, inflight = Inflight})
-         State = #state{next_msg_id = MsgId, inflight = Inflight})
     when QoS =:= ?QOS1 orelse QoS =:= ?QOS2 ->
     case emqx_inflight:is_full(Inflight) of
         true  ->
             enqueue_msg(Msg, State);
         false ->
-            Msg1 = emqx_message:set_header(packet_id, MsgId, Msg),
+            deliver(PacketId, Msg, State),
-            deliver(Msg1, State),
+            await(PacketId, Msg, next_pkt_id(State))
-            await(Msg1, next_msg_id(State))
     end.
 
 enqueue_msg(Msg, State = #state{mqueue = Q}) ->
     inc_stats(enqueue_msg),
-    State#state{mqueue = ?MQueue:in(Msg, Q)}.
+    State#state{mqueue = emqx_mqueue:in(Msg, Q)}.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Deliver
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
-redeliver(Msg = #message{qos = QoS}, State) ->
+redeliver({PacketId, Msg = #message{qos = QoS}}, State) ->
-    deliver(if QoS =:= ?QOS2 -> Msg; true -> emqx_message:set_flag(dup, Msg) end, State);
+    deliver(PacketId, if QoS =:= ?QOS2 -> Msg; true -> emqx_message:set_flag(dup, Msg) end, State);
 
 redeliver({pubrel, PacketId}, #state{client_pid = Pid}) ->
-    Pid ! {redeliver, {?PUBREL, PacketId}}.
+    Pid ! {deliver, {pubrel, PacketId}}.
 
-deliver(Msg, #state{client_pid = Pid, binding = local}) ->
+deliver(PacketId, Msg, #state{client_pid = Pid, binding = local}) ->
-    inc_stats(deliver_msg), Pid ! {deliver, Msg};
+    inc_stats(deliver_msg), Pid ! {deliver, {publish, PacketId, Msg}};
-deliver(Msg, #state{client_pid = Pid, binding = remote}) ->
+deliver(PacketId, Msg, #state{client_pid = Pid, binding = remote}) ->
-    inc_stats(deliver_msg), emqx_rpc:cast(node(Pid), erlang, send, [Pid, {deliver, Msg}]).
+    inc_stats(deliver_msg), emqx_rpc:cast(node(Pid), erlang, send, [Pid, {deliver, PacketId, Msg}]).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Awaiting ACK for QoS1/QoS2 Messages
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
-await(Msg = #message{headers = #{packet_id := PacketId}},
+await(PacketId, Msg, State = #state{inflight       = Inflight,
-      State = #state{inflight       = Inflight,
                                     retry_timer    = RetryTimer,
                                     retry_interval = Interval}) ->
     %% Start retry timer if the Inflight is still empty
     State1 = case RetryTimer == undefined of
-                 true  -> State#state{retry_timer = start_timer(Interval, retry_delivery)};
+                 true  -> State#state{retry_timer = emqx_misc:start_timer(Interval, retry_delivery)};
                  false -> State
              end,
-    State1#state{inflight = emqx_inflight:insert(PacketId, {publish, Msg, os:timestamp()}, Inflight)}.
+    State1#state{inflight = emqx_inflight:insert(PacketId, {publish, {PacketId, Msg}, os:timestamp()}, Inflight)}.
 
 acked(puback, PacketId, State = #state{client_id = ClientId,
                                        username  = Username,
@@ -735,9 +729,9 @@ acked(pubrec, PacketId, State = #state{client_id = ClientId,
 acked(pubcomp, PacketId, State = #state{inflight = Inflight}) ->
     State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Dequeue
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 %% Do nothing if client is disconnected
 dequeue(State = #state{client_pid = undefined}) ->
@@ -750,7 +744,7 @@ dequeue(State = #state{inflight = Inflight}) ->
     end.
 
 dequeue2(State = #state{mqueue = Q}) ->
-    case ?MQueue:out(Q) of
+    case emqx_mqueue:out(Q) of
         {empty, _Q} ->
             State;
         {{value, Msg}, Q1} ->
@@ -758,9 +752,8 @@ dequeue2(State = #state{mqueue = Q}) ->
             dequeue(dispatch(Msg, State#state{mqueue = Q1}))
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Tune QoS
-%%--------------------------------------------------------------------
 
 tune_qos(Topic, Msg = #message{qos = PubQoS},
          #state{subscriptions = SubMap, upgrade_qos = UpgradeQoS}) ->
@@ -775,26 +768,23 @@ tune_qos(Topic, Msg = #message{qos = PubQoS},
             Msg
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Reset Dup
-%%--------------------------------------------------------------------
 
 reset_dup(Msg) ->
     emqx_message:unset_flag(dup, Msg).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% Next Msg Id
-%%--------------------------------------------------------------------
 
-next_msg_id(State = #state{next_msg_id = 16#FFFF}) ->
+next_pkt_id(State = #state{next_pkt_id = 16#FFFF}) ->
-    State#state{next_msg_id = 1};
+    State#state{next_pkt_id = 1};
 
-next_msg_id(State = #state{next_msg_id = Id}) ->
+next_pkt_id(State = #state{next_pkt_id = Id}) ->
-    State#state{next_msg_id = Id + 1}.
+    State#state{next_pkt_id = Id + 1}.
 
 %%--------------------------------------------------------------------
 %% Emit session stats
-%%--------------------------------------------------------------------
 
 emit_stats(State = #state{enable_stats = false}) ->
     State;
@@ -806,7 +796,6 @@ inc_stats(Key) -> put(Key, get(Key) + 1).
 
 %%--------------------------------------------------------------------
 %% Helper functions
-%%--------------------------------------------------------------------
 
 reply(Reply, State) ->
     {reply, Reply, State, hibernate}.

src/emqx_sm.erl

@@ -46,7 +46,7 @@ start_link() ->
     gen_server:start_link({local, ?SM}, ?MODULE, [], []).
 
 %% @doc Open a session.
--spec(open_session(map()) -> {ok, pid()} | {error, term()}).
+-spec(open_session(map()) -> {ok, pid(), boolean()} | {error, term()}).
 open_session(Attrs = #{clean_start := true,
                        client_id   := ClientId,
                        client_pid  := ClientPid}) ->

src/emqx_sm_registry.erl

@@ -22,7 +22,8 @@
 -export([is_enabled/0]).
 -export([register_session/1, lookup_session/1, unregister_session/1]).
 %% gen_server callbacks
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
+         code_change/3]).
 
 -define(REGISTRY, ?MODULE).
 -define(TAB, emqx_session_registry).

src/emqx_sys.erl

@@ -171,6 +171,8 @@ publish(metrics, Metrics) ->
 safe_publish(Topic, Payload) ->
     safe_publish(Topic, #{}, Payload).
 safe_publish(Topic, Flags, Payload) ->
-    Flags1 = maps:merge(#{sys => true}, Flags),
+    emqx_broker:safe_publish(
-    emqx_broker:safe_publish(emqx_message:new(?SYS, Flags1, Topic, iolist_to_binary(Payload))).
+      emqx_message:set_flags(
+        maps:merge(#{sys => true}, Flags),
+        emqx_message:make(?SYS, Topic, iolist_to_binary(Payload)))).
 

src/emqx_sys_mon.erl

@@ -43,7 +43,7 @@ init([Opts]) ->
     {ok, start_timer(#state{events = []})}.
 
 start_timer(State) ->
-    State#state{timer = emqx_misc:start_timer(timer:seconds(2), tick)}.
+    State#state{timer = emqx_misc:start_timer(timer:seconds(2), reset)}.
 
 parse_opt(Opts) ->
     parse_opt(Opts, []).
@@ -126,7 +126,7 @@ handle_info({monitor, SusPid, busy_dist_port, Port}, State) ->
                  safe_publish(busy_dist_port, WarnMsg)
              end, State);
 
-handle_info(reset, State) ->
+handle_info({timeout, _Ref, reset}, State) ->
     {noreply, State#state{events = []}, hibernate};
 
 handle_info(Info, State) ->
@@ -158,5 +158,5 @@ safe_publish(Event, WarnMsg) ->
     emqx_broker:safe_publish(sysmon_msg(Topic, iolist_to_binary(WarnMsg))).
 
 sysmon_msg(Topic, Payload) ->
-    emqx_message:new(?SYSMON, #{sys => true}, Topic, Payload).
+    emqx_message:make(?SYSMON, #{sys => true}, Topic, Payload).
 

src/emqx_topic.erl

@@ -25,10 +25,9 @@
 
 -type(word() :: '' | '+' | '#' | binary()).
 -type(words() :: list(word())).
--type(option() :: {qos, mqtt_qos()} | {share, '$queue' | binary()}).
 -type(triple() :: {root | binary(), word(), binary()}).
 
--export_type([option/0, word/0, triple/0]).
+-export_type([word/0, triple/0]).
 
 -define(MAX_TOPIC_LEN, 4096).
 
@@ -163,20 +162,21 @@ join(Words) ->
                 end, {true, <<>>}, [bin(W) || W <- Words]),
     Bin.
 
--spec(parse(topic()) -> {topic(), [option()]}).
+-spec(parse(topic()) -> {topic(), #{}}).
 parse(Topic) when is_binary(Topic) ->
-    parse(Topic, []).
+    parse(Topic, #{}).
 
 parse(Topic = <<"$queue/", Topic1/binary>>, Options) ->
-    case lists:keyfind(share, 1, Options) of
+    case maps:find(share, Options) of
-        {share, _} -> error({invalid_topic, Topic});
+        {ok, _} -> error({invalid_topic, Topic});
-        false      -> parse(Topic1, [{share, '$queue'} | Options])
+        error   -> parse(Topic1, maps:put(share, '$queue', Options))
     end;
 parse(Topic = <<"$share/", Topic1/binary>>, Options) ->
-    case lists:keyfind(share, 1, Options) of
+    case maps:find(share, Options) of
-        {share, _} -> error({invalid_topic, Topic});
+        {ok, _} -> error({invalid_topic, Topic});
-        false      -> [Group, Topic2] = binary:split(Topic1, <<"/">>),
+        error   -> [Group, Topic2] = binary:split(Topic1, <<"/">>),
-                      {Topic2, [{share, Group} | Options]}
+                   {Topic2, maps:put(share, Group, Options)}
     end;
-parse(Topic, Options) -> {Topic, Options}.
+parse(Topic, Options) ->
+    {Topic, Options}.
 

src/emqx_tracer.erl

@@ -19,6 +19,7 @@
 -include("emqx.hrl").
 
 -export([start_link/0]).
+-export([trace/2]).
 -export([start_trace/2, lookup_traces/0, stop_trace/1]).
 
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
@@ -31,14 +32,17 @@
 -define(TRACER, ?MODULE).
 -define(OPTIONS, [{formatter_config, [time, " [",severity,"] ", message, "\n"]}]).
 
-%%------------------------------------------------------------------------------
-%% Start the tracer
-%%------------------------------------------------------------------------------
-
 -spec(start_link() -> {ok, pid()} | ignore | {error, term()}).
 start_link() ->
     gen_server:start_link({local, ?TRACER}, ?MODULE, [], []).
 
+trace(publish, #message{topic = <<"$SYS/", _/binary>>}) ->
+    %% Dont' trace '$SYS' publish
+    ignore;
+trace(publish, #message{from = From, topic = Topic, payload = Payload})
+    when is_binary(From); is_atom(From) ->
+    emqx_logger:info([{client, From}, {topic, Topic}], "~s PUBLISH to ~s: ~p", [From, Topic, Payload]).
+
 %%------------------------------------------------------------------------------
 %% Start/Stop trace
 %%------------------------------------------------------------------------------

src/emqx_vm.erl

@@ -17,21 +17,15 @@
 -module(emqx_vm).
 
 -export([schedulers/0]).
-
 -export([microsecs/0]).
-
 -export([loads/0, get_system_info/0, get_system_info/1,  mem_info/0, scheduler_usage/1]).
-
 -export([get_memory/0]).
-
 -export([get_process_list/0, get_process_info/0, get_process_info/1,
          get_process_gc/0, get_process_gc/1,
          get_process_group_leader_info/1,
          get_process_limit/0]).
-
 -export([get_ets_list/0, get_ets_info/0, get_ets_info/1,
          get_ets_object/0, get_ets_object/1]).
-
 -export([get_port_types/0, get_port_info/0, get_port_info/1]).
 
 -define(UTIL_ALLOCATORS, [temp_alloc,

src/emqx_ws_connection.erl

@@ -1,18 +1,16 @@
-%%%===================================================================
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved.
+%%
-%%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
-%%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
-%%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
-%%% You may obtain a copy of the License at
+%%
-%%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
-%%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
-%%%
+%% Unless required by applicable law or agreed to in writing, software
-%%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
-%%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
-%%% See the License for the specific language governing permissions and
+%% limitations under the License.
-%%% limitations under the License.
-%%%===================================================================
 
 -module(emqx_ws_connection).
 
@@ -157,8 +155,8 @@ handle_cast({received, Packet}, State = #wsclient_state{proto_state = ProtoState
     end;
 
 handle_cast(Msg, State) ->
-    ?WSLOG(error, "Unexpected Msg: ~p", [Msg], State),
+    ?WSLOG(error, "unexpected msg: ~p", [Msg], State),
-    {noreply, State, hibernate}.
+    {noreply, State}.
 
 handle_info({subscribe, TopicTable}, State) ->
     with_proto(
@@ -172,10 +170,17 @@ handle_info({unsubscribe, Topics}, State) ->
           emqx_protocol:unsubscribe(Topics, ProtoState)
       end, State);
 
-handle_info({suback, PacketId, GrantedQos}, State) ->
+handle_info({suback, PacketId, ReasonCodes}, State) ->
     with_proto(
       fun(ProtoState) ->
-          Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
+          Packet = ?SUBACK_PACKET(PacketId, ReasonCodes),
+          emqx_protocol:send(Packet, ProtoState)
+      end, State);
+
+handle_info({unsuback, PacketId, ReasonCodes}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          Packet = ?UNSUBACK_PACKET(PacketId, ReasonCodes),
           emqx_protocol:send(Packet, ProtoState)
       end, State);
 

src/emqx_zone.erl

@@ -0,0 +1,78 @@
+%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(emqx_zone).
+
+-behaviour(gen_server).
+
+-export([start_link/0]).
+-export([get_env/2, get_env/3]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
+         code_change/3]).
+
+-record(state, {timer}).
+-define(TAB, ?MODULE).
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+
+get_env(Zone, Par) ->
+    get_env(Zone, Par, undefined).
+
+get_env(Zone, Par, Def) ->
+    try ets:lookup_element(?TAB, {Zone, Par}, 2) catch error:badarg -> Def end.
+
+%%------------------------------------------------------------------------------
+%% gen_server callbacks
+%%------------------------------------------------------------------------------
+
+init([]) ->
+    _ = emqx_tables:new(?TAB, [set, {read_concurrency, true}]),
+    {ok, element(2, handle_info(reload, #state{}))}.
+
+handle_call(Req, _From, State) ->
+    emqx_logger:error("[Zone] unexpected call: ~p", [Req]),
+    {reply, ignored, State}.
+
+handle_cast(Msg, State) ->
+    emqx_logger:error("[Zone] unexpected cast: ~p", [Msg]),
+    {noreply, State}.
+
+handle_info(reload, State) ->
+    lists:foreach(
+      fun({Zone, Options}) ->
+          [ets:insert(?TAB, {{Zone, Par}, Val}) || {Par, Val} <- Options]
+      end, emqx_config:get_env(zones, [])),
+    {noreply, ensure_reload_timer(State), hibernate};
+
+handle_info(Info, State) ->
+    emqx_logger:error("[Zone] unexpected info: ~p", [Info]),
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+ensure_reload_timer(State) ->
+    State#state{timer = erlang:send_after(5000, self(), reload)}.
+