Merge pull request #905 from emqtt/emq20

Version 2.1 - Improve the design of Session and Inflight

Makefile

@@ -1,14 +1,15 @@
 PROJECT = emqttd
 PROJECT_DESCRIPTION = Erlang MQTT Broker
-PROJECT_VERSION = 2.0.7
+PROJECT_VERSION = 2.1
 
-DEPS = gproc lager esockd mochiweb
+DEPS = gproc lager esockd mochiweb lager_syslog
 
 dep_gproc       = git https://github.com/uwiger/gproc
 dep_getopt      = git https://github.com/jcomellas/getopt v0.8.2
 dep_lager       = git https://github.com/basho/lager master
 dep_esockd      = git https://github.com/emqtt/esockd master
 dep_mochiweb    = git https://github.com/emqtt/mochiweb
+dep_lager_syslog  = git https://github.com/basho/lager_syslog
 
 ERLC_OPTS += +'{parse_transform, lager_transform}'
 
@@ -24,9 +25,9 @@ TEST_ERLC_OPTS += +'{parse_transform, lager_transform}'
 EUNIT_OPTS = verbose
 # EUNIT_ERL_OPTS =
 
-CT_SUITES = emqttd emqttd_access emqttd_lib emqttd_mod emqttd_net \
+CT_SUITES = emqttd emqttd_access emqttd_lib emqttd_inflight emqttd_mod \
-            emqttd_mqueue emqttd_protocol emqttd_topic emqttd_trie \
+            emqttd_net emqttd_mqueue emqttd_protocol emqttd_topic \
-            emqttd_vm
+            emqttd_trie emqttd_vm
 
 CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqttd_ct@127.0.0.1
 

etc/emq.conf

@@ -1,3 +1,8 @@
+
+##===================================================================
+## EMQ Configuration R2.1
+##===================================================================
+
 ##--------------------------------------------------------------------
 ## Node Args
 ##--------------------------------------------------------------------
@@ -45,16 +50,25 @@ node.crash_dump = {{ platform_log_dir }}/crash.dump
 node.dist_net_ticktime = 60
 
 ## Distributed node port range
-## node.dist_listen_min = 6000
+## node.dist_listen_min = 6369
-## node.dist_listen_max = 6999
+## node.dist_listen_max = 6369
 
 ##--------------------------------------------------------------------
 ## Log
 ##--------------------------------------------------------------------
 
+## Set the log dir
+log.dir = {{ platform_log_dir }}
+
 ## Console log. Enum: off, file, console, both
 log.console = console
 
+## Syslog. Enum: on, off
+log.syslog = on
+
+##  syslog level. Enum: debug, info, notice, warning, error, critical, alert, emergency
+log.syslog.level = error
+
 ## Console log level. Enum: debug, info, notice, warning, error, critical, alert, emergency
 log.console.level = error
 
@@ -70,18 +84,9 @@ log.crash = on
 log.crash.file = {{ platform_log_dir }}/crash.log
 
 ##--------------------------------------------------------------------
-## MQTT Protocol
+## Allow Anonymous and Default ACL
 ##--------------------------------------------------------------------
 
-## Max ClientId Length Allowed.
-mqtt.max_clientid_len = 1024
-
-## Max Packet Size Allowed, 64K by default.
-mqtt.max_packet_size = 64KB
-
-## Client Idle Timeout (Second)
-mqtt.client_idle_timeout = 30
-
 ## Allow Anonymous authentication
 mqtt.allow_anonymous = true
 
@@ -91,25 +96,48 @@ mqtt.acl_file = {{ platform_etc_dir }}/acl.conf
 ## Cache ACL for PUBLISH
 mqtt.cache_acl = true
 
+##--------------------------------------------------------------------
+## MQTT Protocol
+##--------------------------------------------------------------------
+
+## Max ClientId Length Allowed.
+mqtt.max_clientid_len = 1024
+
+## Max Packet Size Allowed, 64K by default.
+mqtt.max_packet_size = 64KB
+
+##--------------------------------------------------------------------
+## MQTT Client
+##--------------------------------------------------------------------
+
+## Client Idle Timeout (Second)
+mqtt.client.idle_timeout = 30s
+
+## Enable client Stats: seconds or off
+mqtt.client.enable_stats = off
+
 ##--------------------------------------------------------------------
 ## MQTT Session
 ##--------------------------------------------------------------------
 
+## Upgrade QoS?
+mqtt.session.upgrade_qos = off
+
 ## Max number of QoS 1 and 2 messages that can be “inflight” at one time.
 ## 0 means no limit
-mqtt.session.max_inflight = 100
+mqtt.session.max_inflight = 32
 
-## Retry interval for redelivering QoS1/2 messages.
+## Retry Interval for redelivering QoS1/2 messages.
-mqtt.session.retry_interval = 60
+mqtt.session.retry_interval = 20s
-
-## Awaiting PUBREL Timeout
-mqtt.session.await_rel_timeout = 20
 
 ## Max Packets that Awaiting PUBREL, 0 means no limit
-mqtt.session.max_awaiting_rel = 0
+mqtt.session.max_awaiting_rel = 100
 
-## Statistics Collection Interval(seconds)
+## Awaiting PUBREL Timeout
-mqtt.session.collect_interval = 0
+mqtt.session.await_rel_timeout = 20s
+
+## Enable Statistics at the Interval(seconds)
+mqtt.session.enable_stats = off
 
 ## Expired after 1 day:
 ## w - week
@@ -117,7 +145,7 @@ mqtt.session.collect_interval = 0
 ## h - hour
 ## m - minute
 ## s - second
-mqtt.session.expired_after = 1d
+mqtt.session.expiry_interval = 2h
 
 ##--------------------------------------------------------------------
 ## MQTT Queue
@@ -212,9 +240,10 @@ mqtt.listener.ssl.max_clients = 512
 ## Rate Limit. Format is 'burst,rate', Unit is KB/Sec
 ## mqtt.listener.ssl.rate_limit = 100,10
 
-## Configuring SSL Options
+## Configuring SSL Options. See http://erlang.org/doc/man/ssl.html
-## See http://erlang.org/doc/man/ssl.html
+### TLS only for POODLE attack
-mqtt.listener.ssl.handshake_timeout = 15
+mqtt.listener.ssl.tls_versions = tlsv1.2,tlsv1.1,tlsv1
+mqtt.listener.ssl.handshake_timeout = 15s
 mqtt.listener.ssl.keyfile = {{ platform_etc_dir }}/certs/key.pem
 mqtt.listener.ssl.certfile = {{ platform_etc_dir }}/certs/cert.pem
 ## mqtt.listener.ssl.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem

include/emqttd.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 %% Banner
 %%--------------------------------------------------------------------
 
--define(COPYRIGHT, "Copyright (C) 2012-2017, Feng Lee <feng@emqtt.io>").
+-define(COPYRIGHT, "Copyright (c) 2013-2017 EMQ Enterprise, Inc.").
 
 -define(LICENSE_MESSAGE, "Licensed under the Apache License, Version 2.0").
 
@@ -48,21 +48,21 @@
 %% MQTT Topic
 %%--------------------------------------------------------------------
 
--record(mqtt_topic, {
+-record(mqtt_topic,
-    topic      :: binary(),
+        { topic      :: binary(),
-    flags = [] :: [retained | static]
+          flags = [] :: [retained | static]
-}).
+        }).
 
 -type(mqtt_topic() :: #mqtt_topic{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Subscription
 %%--------------------------------------------------------------------
--record(mqtt_subscription, {
+-record(mqtt_subscription,
-    subid :: binary() | atom(),
+        { subid :: binary() | atom(),
-    topic :: binary(),
+          topic :: binary(),
-    qos   :: 0 | 1 | 2
+          qos   :: 0 | 1 | 2
-}).
+        }).
 
 -type(mqtt_subscription() :: #mqtt_subscription{}).
 
@@ -73,18 +73,18 @@
 -type(ws_header_key() :: atom() | binary() | string()).
 -type(ws_header_val() :: atom() | binary() | string() | integer()).
 
--record(mqtt_client, {
+-record(mqtt_client,
-    client_id     :: binary() | undefined,
+        { client_id     :: binary() | undefined,
-    client_pid    :: pid(),
+          client_pid    :: pid(),
-    username      :: binary() | undefined,
+          username      :: binary() | undefined,
-    peername      :: {inet:ip_address(), integer()},
+          peername      :: {inet:ip_address(), inet:port_number()},
-    clean_sess    :: boolean(),
+          clean_sess    :: boolean(),
-    proto_ver     :: 3 | 4,
+          proto_ver     :: 3 | 4,
-    keepalive = 0,
+          keepalive = 0,
-    will_topic    :: undefined | binary(),
+          will_topic    :: undefined | binary(),
-    ws_initial_headers :: list({ws_header_key(), ws_header_val()}),
+          ws_initial_headers :: list({ws_header_key(), ws_header_val()}),
-    connected_at  :: erlang:timestamp()
+          connected_at  :: erlang:timestamp()
-}).
+        }).
 
 -type(mqtt_client() :: #mqtt_client{}).
 
@@ -92,33 +92,46 @@
 %% MQTT Session
 %%--------------------------------------------------------------------
 
--record(mqtt_session, {
+-record(mqtt_session,
-    client_id  :: binary(),
+        { client_id  :: binary(),
-    sess_pid   :: pid(),
+          sess_pid   :: pid(),
-    persistent :: boolean()
+          clean_sess :: boolean()
-}).
+        }).
 
 -type(mqtt_session() :: #mqtt_session{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Message
 %%--------------------------------------------------------------------
+
 -type(mqtt_msgid() :: binary() | undefined).
+
 -type(mqtt_pktid() :: 1..16#ffff | undefined).
 
--record(mqtt_message, {
+-record(mqtt_message,
-    id              :: mqtt_msgid(),         %% Global unique message ID
+        { %% Global unique message ID
-    pktid           :: mqtt_pktid(),         %% PacketId
+          id              :: mqtt_msgid(),
-    from            :: {binary(), undefined | binary()}, %% ClientId and Username
+          %% PacketId
-    topic           :: binary(),             %% Topic that the message is published to
+          pktid           :: mqtt_pktid(),
-    qos     = 0     :: 0 | 1 | 2,            %% Message QoS
+          %% ClientId and Username
-    flags   = []    :: [retain | dup | sys], %% Message Flags
+          from            :: {binary(), undefined | binary()},
-    retain  = false :: boolean(),            %% Retain flag
+          %% Topic that the message is published to
-    dup     = false :: boolean(),            %% Dup flag
+          topic           :: binary(),
-    sys     = false :: boolean(),            %% $SYS flag
+          %% Message QoS
-    headers = []    :: list(),
+          qos     = 0     :: 0 | 1 | 2,
-    payload         :: binary(),             %% Payload
+          %% Message Flags
-    timestamp       :: pos_integer()         %% os:timestamp to seconds
+          flags   = []    :: [retain | dup | sys],
+          %% Retain flag
+          retain  = false :: boolean(),
+          %% Dup flag
+          dup     = false :: boolean(),
+          %% $SYS flag
+          sys     = false :: boolean(),
+          headers = []    :: list(),
+          %% Payload
+          payload         :: binary(),
+          %% Timestamp
+          timestamp       :: erlang:timestamp()
 }).
 
 -type(mqtt_message() :: #mqtt_message{}).
@@ -126,46 +139,45 @@
 %%--------------------------------------------------------------------
 %% MQTT Delivery
 %%--------------------------------------------------------------------
--record(mqtt_delivery, {
+
-    sender  :: pid(),          %% Pid of the sender/publisher
+-record(mqtt_delivery,
-    message :: mqtt_message(), %% Message
+        { sender  :: pid(),          %% Pid of the sender/publisher
-    flows   :: list()
+          message :: mqtt_message(), %% Message
-}).
+          flows   :: list()
+        }).
 
 -type(mqtt_delivery() :: #mqtt_delivery{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Route
 %%--------------------------------------------------------------------
--record(mqtt_route, {
+
-    topic   :: binary(),
+-record(mqtt_route,
-    node    :: node()
+        { topic   :: binary(),
-}).
+          node    :: node()
+        }).
 
 -type(mqtt_route() :: #mqtt_route{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Alarm
 %%--------------------------------------------------------------------
--record(mqtt_alarm, {
+
-    id          :: binary(),
+-record(mqtt_alarm,
-    severity    :: warning | error | critical,
+        { id          :: binary(),
-    title       :: iolist() | binary(),
+          severity    :: warning | error | critical,
-    summary     :: iolist() | binary(),
+          title       :: iolist() | binary(),
-    timestamp   :: erlang:timestamp() %% Timestamp
+          summary     :: iolist() | binary(),
-}).
+          timestamp   :: erlang:timestamp() %% Timestamp
+        }).
 
 -type(mqtt_alarm() :: #mqtt_alarm{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Plugin
 %%--------------------------------------------------------------------
--record(mqtt_plugin, {
+
-    name,
+-record(mqtt_plugin, { name, version, descr, active = false }).
-    version,
-    descr,
-    active = false
-}).
 
 -type(mqtt_plugin() :: #mqtt_plugin{}).
 
@@ -173,14 +185,8 @@
 %% MQTT CLI Command
 %% For example: 'broker metrics'
 %%--------------------------------------------------------------------
--record(mqtt_cli, {
+
-    name,
+-record(mqtt_cli, { name, action, args = [], opts = [], usage, descr }).
-    action,
-    args = [],
-    opts = [],
-    usage,
-    descr
-}).
 
 -type(mqtt_cli() :: #mqtt_cli{}).
 

include/emqttd_cli.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

include/emqttd_internal.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

include/emqttd_protocol.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,23 +14,32 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% MQTT Protocol Header
+%%--------------------------------------------------------------------
+%% MQTT SockOpts
+%%--------------------------------------------------------------------
+
+-define(MQTT_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true},
+                        {backlog, 512}, {nodelay, true}]).
 
 %%--------------------------------------------------------------------
 %% MQTT Protocol Version and Levels
 %%--------------------------------------------------------------------
--define(MQTT_PROTO_V31,  3).
+
--define(MQTT_PROTO_V311, 4).
+-define(MQTT_PROTO_V3, 3).
+-define(MQTT_PROTO_V4, 4).
+-define(MQTT_PROTO_V5, 5).
 
 -define(PROTOCOL_NAMES, [
-    {?MQTT_PROTO_V31,  <<"MQIsdp">>},
+    {?MQTT_PROTO_V3, <<"MQIsdp">>},
-    {?MQTT_PROTO_V311, <<"MQTT">>}]).
+    {?MQTT_PROTO_V4, <<"MQTT">>},
+    {?MQTT_PROTO_V5, <<"MQTT">>}]).
 
--type(mqtt_vsn() :: ?MQTT_PROTO_V31 | ?MQTT_PROTO_V311).
+-type(mqtt_vsn() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 | ?MQTT_PROTO_V5).
 
 %%--------------------------------------------------------------------
-%% MQTT QoS
+%% MQTT QoS Level
 %%--------------------------------------------------------------------
+
 -define(QOS_0, 0). %% At most once
 -define(QOS_1, 1). %% At least once
 -define(QOS_2, 2). %% Exactly once
@@ -63,28 +72,30 @@
     end).
 
 %%--------------------------------------------------------------------
-%% Max ClientId Length. Why 1024? NiDongDe...
+%% Max ClientId Length. Why 1024?
 %%--------------------------------------------------------------------
+
 -define(MAX_CLIENTID_LEN, 1024).
 
 %%--------------------------------------------------------------------
 %% MQTT Control Packet Types
 %%--------------------------------------------------------------------
--define(RESERVED,     0).   %% Reserved
+-define(RESERVED,     0). %% Reserved
--define(CONNECT,      1).   %% Client request to connect to Server
+-define(CONNECT,      1). %% Client request to connect to Server
--define(CONNACK,      2).   %% Server to Client: Connect acknowledgment
+-define(CONNACK,      2). %% Server to Client: Connect acknowledgment
--define(PUBLISH,      3).   %% Publish message
+-define(PUBLISH,      3). %% Publish message
--define(PUBACK,       4).   %% Publish acknowledgment
+-define(PUBACK,       4). %% Publish acknowledgment
--define(PUBREC,       5).   %% Publish received (assured delivery part 1)
+-define(PUBREC,       5). %% Publish received (assured delivery part 1)
--define(PUBREL,       6).   %% Publish release (assured delivery part 2)
+-define(PUBREL,       6). %% Publish release (assured delivery part 2)
--define(PUBCOMP,      7).   %% Publish complete (assured delivery part 3)
+-define(PUBCOMP,      7). %% Publish complete (assured delivery part 3)
--define(SUBSCRIBE,    8).   %% Client subscribe request
+-define(SUBSCRIBE,    8). %% Client subscribe request
--define(SUBACK,       9).   %% Server Subscribe acknowledgment
+-define(SUBACK,       9). %% Server Subscribe acknowledgment
--define(UNSUBSCRIBE, 10).   %% Unsubscribe request
+-define(UNSUBSCRIBE, 10). %% Unsubscribe request
--define(UNSUBACK,    11).   %% Unsubscribe acknowledgment
+-define(UNSUBACK,    11). %% Unsubscribe acknowledgment
--define(PINGREQ,     12).   %% PING request
+-define(PINGREQ,     12). %% PING request
--define(PINGRESP,    13).   %% PING response
+-define(PINGRESP,    13). %% PING response
--define(DISCONNECT,  14).   %% Client is disconnecting
+-define(DISCONNECT,  14). %% Client is disconnecting
+-define(AUTH,        15). %% Authentication exchange
 
 -define(TYPE_NAMES, [
     'CONNECT',
@@ -100,25 +111,28 @@
     'UNSUBACK',
     'PINGREQ',
     'PINGRESP',
-    'DISCONNECT']).
+    'DISCONNECT',
+    'AUTH']).
 
 -type(mqtt_packet_type() :: ?RESERVED..?DISCONNECT).
 
 %%--------------------------------------------------------------------
 %% MQTT Connect Return Codes
 %%--------------------------------------------------------------------
--define(CONNACK_ACCEPT,      0).    %% Connection accepted
+
--define(CONNACK_PROTO_VER,   1).    %% Unacceptable protocol version
+-define(CONNACK_ACCEPT,      0). %% Connection accepted
--define(CONNACK_INVALID_ID,  2).    %% Client Identifier is correct UTF-8 but not allowed by the Server
+-define(CONNACK_PROTO_VER,   1). %% Unacceptable protocol version
--define(CONNACK_SERVER,      3).    %% Server unavailable
+-define(CONNACK_INVALID_ID,  2). %% Client Identifier is correct UTF-8 but not allowed by the Server
--define(CONNACK_CREDENTIALS, 4).    %% Username or password is malformed
+-define(CONNACK_SERVER,      3). %% Server unavailable
--define(CONNACK_AUTH,        5).    %% Client is not authorized to connect
+-define(CONNACK_CREDENTIALS, 4). %% Username or password is malformed
+-define(CONNACK_AUTH,        5). %% Client is not authorized to connect
 
 -type(mqtt_connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH).
 
 %%--------------------------------------------------------------------
 %% MQTT Parser and Serializer
 %%--------------------------------------------------------------------
+
 -define(MAX_LEN, 16#fffffff).
 -define(HIGHBIT, 2#10000000).
 -define(LOWBITS, 2#01111111).
@@ -126,76 +140,87 @@
 %%--------------------------------------------------------------------
 %% MQTT Packet Fixed Header
 %%--------------------------------------------------------------------
+
 -record(mqtt_packet_header, {
-    type   = ?RESERVED  :: mqtt_packet_type(),
+    type   = ?RESERVED :: mqtt_packet_type(),
-    dup    = false      :: boolean(),
+    dup    = false     :: boolean(),
-    qos    = ?QOS_0     :: mqtt_qos(),
+    qos    = ?QOS_0    :: mqtt_qos(),
-    retain = false      :: boolean()}).
+    retain = false     :: boolean()}).
 
 %%--------------------------------------------------------------------
 %% MQTT Packets
 %%--------------------------------------------------------------------
--type(mqtt_client_id()  :: binary()).
--type(mqtt_username()   :: binary() | undefined).
--type(mqtt_packet_id()  :: 1..16#ffff | undefined).
 
--record(mqtt_packet_connect,  {
+-type(mqtt_client_id() :: binary()).
-    client_id   = <<>>              :: mqtt_client_id(),
+-type(mqtt_username()  :: binary() | undefined).
-    proto_ver   = ?MQTT_PROTO_V311  :: mqtt_vsn(),
+-type(mqtt_packet_id() :: 1..16#ffff | undefined).
-    proto_name  = <<"MQTT">>        :: binary(),
-    will_retain = false             :: boolean(),
-    will_qos    = ?QOS_0            :: mqtt_qos(),
-    will_flag   = false             :: boolean(),
-    clean_sess  = false             :: boolean(),
-    keep_alive  = 60                :: non_neg_integer(),
-    will_topic  = undefined         :: undefined | binary(),
-    will_msg    = undefined         :: undefined | binary(),
-    username    = undefined         :: undefined | binary(),
-    password    = undefined         :: undefined | binary()}).
 
--record(mqtt_packet_connack, {
+-record(mqtt_packet_connect,
-    ack_flags = ?RESERVED   :: 0 | 1,
+        { client_id   = <<>>           :: mqtt_client_id(),
-    return_code             :: mqtt_connack() }).
+          proto_ver   = ?MQTT_PROTO_V4 :: mqtt_vsn(),
+          proto_name  = <<"MQTT">>     :: binary(),
+          will_retain = false          :: boolean(),
+          will_qos    = ?QOS_0         :: mqtt_qos(),
+          will_flag   = false          :: boolean(),
+          clean_sess  = false          :: boolean(),
+          keep_alive  = 60             :: non_neg_integer(),
+          will_topic  = undefined      :: undefined | binary(),
+          will_msg    = undefined      :: undefined | binary(),
+          username    = undefined      :: undefined | binary(),
+          password    = undefined      :: undefined | binary()
+        }).
 
--record(mqtt_packet_publish, {
+-record(mqtt_packet_connack,
-    topic_name  :: binary(),
+        { ack_flags = ?RESERVED :: 0 | 1,
-    packet_id   :: mqtt_packet_id() }).
+          return_code           :: mqtt_connack()
+        }).
 
--record(mqtt_packet_puback, {
+-record(mqtt_packet_publish,
-    packet_id   :: mqtt_packet_id() }).
+        { topic_name :: binary(),
+          packet_id  :: mqtt_packet_id()
+        }).
 
--record(mqtt_packet_subscribe, {
+-record(mqtt_packet_puback,
-    packet_id   :: mqtt_packet_id(),
+        { packet_id :: mqtt_packet_id() }).
-    topic_table :: list({binary(), mqtt_qos()}) }).
 
--record(mqtt_packet_unsubscribe, {
+-record(mqtt_packet_subscribe,
-    packet_id   :: mqtt_packet_id(),
+        { packet_id   :: mqtt_packet_id(),
-    topics      :: list(binary()) }).
+          topic_table :: list({binary(), mqtt_qos()})
+        }).
 
--record(mqtt_packet_suback, {
+-record(mqtt_packet_unsubscribe,
-    packet_id   :: mqtt_packet_id(),
+        { packet_id :: mqtt_packet_id(),
-    qos_table   :: list(mqtt_qos() | 128) }).
+          topics    :: list(binary())
+        }).
 
--record(mqtt_packet_unsuback, {
+-record(mqtt_packet_suback,
-    packet_id   :: mqtt_packet_id() }).
+        { packet_id   :: mqtt_packet_id(),
+          qos_table   :: list(mqtt_qos() | 128)
+        }).
+
+-record(mqtt_packet_unsuback,
+        { packet_id   :: mqtt_packet_id() }).
 
 %%--------------------------------------------------------------------
 %% MQTT Control Packet
 %%--------------------------------------------------------------------
--record(mqtt_packet, {
-    header    :: #mqtt_packet_header{},
-    variable  :: #mqtt_packet_connect{} | #mqtt_packet_connack{}
-                | #mqtt_packet_publish{} | #mqtt_packet_puback{}
-                | #mqtt_packet_subscribe{} | #mqtt_packet_suback{}
-                | #mqtt_packet_unsubscribe{} | #mqtt_packet_unsuback{}
-                | mqtt_packet_id() | undefined,
-    payload   :: binary() | undefined }).
 
--type mqtt_packet() :: #mqtt_packet{}.
+-record(mqtt_packet,
+        { header    :: #mqtt_packet_header{},
+          variable  :: #mqtt_packet_connect{} | #mqtt_packet_connack{}
+                     | #mqtt_packet_publish{} | #mqtt_packet_puback{}
+                     | #mqtt_packet_subscribe{} | #mqtt_packet_suback{}
+                     | #mqtt_packet_unsubscribe{} | #mqtt_packet_unsuback{}
+                     | mqtt_packet_id() | undefined,
+          payload   :: binary() | undefined
+        }).
+
+-type(mqtt_packet() :: #mqtt_packet{}).
 
 %%--------------------------------------------------------------------
 %% MQTT Packet Match
 %%--------------------------------------------------------------------
+
 -define(CONNECT_PACKET(Var),
     #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}, variable = Var}).
 

include/emqttd_trie.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2016-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,20 +16,20 @@
 
 -type(trie_node_id() :: binary() | atom()).
 
--record(trie_node, {
+-record(trie_node,
-    node_id         :: trie_node_id(),
+        { node_id        :: trie_node_id(),
-    edge_count = 0  :: non_neg_integer(),
+          edge_count = 0 :: non_neg_integer(),
-    topic           :: binary() | undefined,
+          topic          :: binary() | undefined,
-    flags           :: [retained | static]
+          flags          :: [retained | static]
-}).
+        }).
 
--record(trie_edge, {
+-record(trie_edge,
-    node_id :: trie_node_id(),
+        { node_id :: trie_node_id(),
-    word    :: binary() | atom()
+          word    :: binary() | atom()
-}).
+        }).
 
--record(trie, {
+-record(trie,
-    edge    :: #trie_edge{},
+        { edge    :: #trie_edge{},
-    node_id :: trie_node_id()
+          node_id :: trie_node_id()
-}).
+        }).
 

priv/emq.schema

@@ -148,8 +148,13 @@ end}.
 %% Log
 %%--------------------------------------------------------------------
 
+{mapping, "log.dir", "lager.log_dir", [
+  {default, "log"},
+  {datatype, string}
+]}.
+
 {mapping, "log.console", "lager.handlers", [
-  {default, file },
+  {default, file},
   {datatype, {enum, [off, file, console, both]}}
 ]}.
 
@@ -168,6 +173,26 @@ end}.
   {datatype, file}
 ]}.
 
+{mapping, "log.syslog", "lager.handlers", [
+  {default,  off},
+  {datatype, flag}
+]}.
+
+{mapping, "log.syslog.identity", "lager.handlers", [
+  {default, "emq"},
+  {datatype, string}
+]}.
+
+{mapping, "log.syslog.facility", "lager.handlers", [
+  {default, local0},
+  {datatype, {enum, [daemon, local0, local1, local2, local3, local4, local5, local6, local7]}}
+]}.
+
+{mapping, "log.syslog.level", "lager.handlers", [
+  {default, err},
+  {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency]}}
+]}.
+
 {mapping, "log.error.redirect", "lager.error_logger_redirect", [
   {default, on},
   {datatype, flag},
@@ -209,7 +234,16 @@ end}.
       both -> [ConsoleHandler, ConsoleFileHandler];
       _ -> []
     end,
-    ConsoleHandlers ++ ErrorHandler
+
+    SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of
+      false -> [];
+      true  -> [{lager_syslog_backend,
+                  [cuttlefish:conf_get("log.syslog.identity", Conf),
+                   cuttlefish:conf_get("log.syslog.facility", Conf),
+                   cuttlefish:conf_get("log.syslog.level", Conf)]}]
+    end,
+
+    ConsoleHandlers ++ ErrorHandler ++ SyslogHandler
   end
 }.
 
@@ -240,33 +274,9 @@ end}.
 ]}.
 
 %%--------------------------------------------------------------------
-%% MQTT Protocol
+%% Allow Anonymous and Default ACL
 %%--------------------------------------------------------------------
 
-%% @doc Set the Max ClientId Length Allowed.
-{mapping, "mqtt.max_clientid_len", "emqttd.protocol", [
-  {default, 1024},
-  {datatype, integer}
-]}.
-
-%% @doc Max Packet Size Allowed, 64K by default.
-{mapping, "mqtt.max_packet_size", "emqttd.protocol", [
-  {default, "64KB"},
-  {datatype, bytesize}
-]}.
-
-%% @doc Client Idle Timeout.
-{mapping, "mqtt.client_idle_timeout", "emqttd.protocol", [
-  {default, 30},
-  {datatype, integer}
-]}.
-
-{translation, "emqttd.protocol", fun(Conf) ->
-  [{max_clientid_len,    cuttlefish:conf_get("mqtt.max_clientid_len", Conf)},
-   {max_packet_size,     cuttlefish:conf_get("mqtt.max_packet_size", Conf)},
-   {client_idle_timeout, cuttlefish:conf_get("mqtt.client_idle_timeout", Conf)}]
-end}.
-
 %% @doc Allow Anonymous
 {mapping, "mqtt.allow_anonymous", "emqttd.allow_anonymous", [
   {default, false},
@@ -285,10 +295,58 @@ end}.
   {datatype, {enum, [true, false]}}
 ]}.
 
+%%--------------------------------------------------------------------
+%% MQTT Protocol
+%%--------------------------------------------------------------------
+
+%% @doc Set the Max ClientId Length Allowed.
+{mapping, "mqtt.max_clientid_len", "emqttd.protocol", [
+  {default, 1024},
+  {datatype, integer}
+]}.
+
+%% @doc Max Packet Size Allowed, 64K by default.
+{mapping, "mqtt.max_packet_size", "emqttd.protocol", [
+  {default, "64KB"},
+  {datatype, bytesize}
+]}.
+
+{translation, "emqttd.protocol", fun(Conf) ->
+  [{max_clientid_len, cuttlefish:conf_get("mqtt.max_clientid_len", Conf)},
+   {max_packet_size,  cuttlefish:conf_get("mqtt.max_packet_size", Conf)}]
+end}.
+
+%%--------------------------------------------------------------------
+%% MQTT Client
+%%--------------------------------------------------------------------
+
+%% @doc Client Idle Timeout.
+{mapping, "mqtt.client.idle_timeout", "emqttd.client", [
+  {default, "30s"},
+  {datatype, {duration, ms}}
+]}.
+
+%% @doc Enable Stats of Client.
+{mapping, "mqtt.client.enable_stats", "emqttd.client", [
+  {default, off},
+  {datatype, [{duration, ms}, flag]}
+]}.
+
+%% @doc Client
+{translation, "emqttd.client", fun(Conf) ->
+  [{client_idle_timeout, cuttlefish:conf_get("mqtt.client.idle_timeout", Conf)},
+   {client_enable_stats, cuttlefish:conf_get("mqtt.client.enable_stats", Conf)}]
+end}.
+
 %%--------------------------------------------------------------------
 %% MQTT Session
 %%--------------------------------------------------------------------
 
+%% @doc Upgrade QoS?
+{mapping, "mqtt.session.upgrade_qos", "emqttd.session", [
+  {default, off},
+  {datatype, flag}
+]}.
 %% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time.
 %% 0 means no limit
 {mapping, "mqtt.session.max_inflight", "emqttd.session", [
@@ -296,17 +354,10 @@ end}.
   {datatype, integer}
 ]}.
 
-
 %% @doc Retry interval for redelivering QoS1/2 messages.
 {mapping, "mqtt.session.retry_interval", "emqttd.session", [
-  {default, 60},
+  {default, "20s"},
-  {datatype, integer}
+  {datatype, {duration, ms}}
-]}.
-
-%% @doc Awaiting PUBREL Timeout
-{mapping, "mqtt.session.await_rel_timeout", "emqttd.session", [
-  {default, 30},
-  {datatype, integer}
 ]}.
 
 %% @doc Max Packets that Awaiting PUBREL, 0 means no limit
@@ -315,25 +366,32 @@ end}.
   {datatype, integer}
 ]}.
 
-%% @doc Statistics Collection Interval(seconds)
+%% @doc Awaiting PUBREL Timeout
-{mapping, "mqtt.session.collect_interval", "emqttd.session", [
+{mapping, "mqtt.session.await_rel_timeout", "emqttd.session", [
-  {default, 0},
+  {default, "20s"},
-  {datatype, integer}
+  {datatype, {duration, ms}}
 ]}.
 
-%% @doc Session expired after...
+%% @doc Enable Stats
-{mapping, "mqtt.session.expired_after", "emqttd.session", [
+{mapping, "mqtt.session.enable_stats", "emqttd.session", [
-  {default, "2d"},
+  {default, off},
-  {datatype, {duration, s}}
+  {datatype, [{duration, ms}, flag]}
+]}.
+
+%% @doc Session Expiry Interval
+{mapping, "mqtt.session.expiry_interval", "emqttd.session", [
+  {default, "2h"},
+  {datatype, {duration, ms}}
 ]}.
 
 {translation, "emqttd.session", fun(Conf) ->
-  [{max_inflight, cuttlefish:conf_get("mqtt.session.max_inflight", Conf)},
+  [{upgrade_qos,       cuttlefish:conf_get("mqtt.session.upgrade_qos", Conf)},
-   {retry_interval, cuttlefish:conf_get("mqtt.session.retry_interval", Conf)},
+   {max_inflight,      cuttlefish:conf_get("mqtt.session.max_inflight", Conf)},
+   {retry_interval,    cuttlefish:conf_get("mqtt.session.retry_interval", Conf)},
+   {max_awaiting_rel,  cuttlefish:conf_get("mqtt.session.max_awaiting_rel", Conf)},
    {await_rel_timeout, cuttlefish:conf_get("mqtt.session.await_rel_timeout", Conf)},
-   {max_awaiting_rel, cuttlefish:conf_get("mqtt.session.max_awaiting_rel", Conf)},
+   {enable_stats,      cuttlefish:conf_get("mqtt.session.enable_stats", Conf)},
-   {collect_interval, cuttlefish:conf_get("mqtt.session.collect_interval", Conf)},
+   {expiry_interval,   cuttlefish:conf_get("mqtt.session.expiry_interval", Conf)}]
-   {expired_after, cuttlefish:conf_get("mqtt.session.expired_after", Conf)}]
 end}.
 
 %%--------------------------------------------------------------------
@@ -528,9 +586,13 @@ end}.
   {datatype, string}
 ]}.
 
+{mapping, "mqtt.listener.ssl.tls_versions", "emqttd.listeners", [
+  {datatype, string}
+]}.
+
 {mapping, "mqtt.listener.ssl.handshake_timeout", "emqttd.listeners", [
-  {default, 15},
+  {default, "15s"},
-  {datatype, integer}
+  {datatype, {duration, ms}}
 ]}.
 
 {mapping, "mqtt.listener.ssl.keyfile", "emqttd.listeners", [
@@ -622,8 +684,16 @@ end}.
                            {buffer,  cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)},
                            {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}])
               end,
+
+    SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end,
+
     SslOpts = fun(Prefix) ->
-                  Filter([{handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf) * 1000},
+                 Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of
+                                undefined -> undefined;
+                                L -> [list_to_atom(V) || V <- L]
+                            end,
+                  Filter([{versions, Versions},
+                          {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf), undefined},
                           {keyfile,    cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},
                           {certfile,   cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
                           {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)},

rebar.config

@@ -1,7 +1,4 @@
 {deps, [
-	{gproc,".*",{git,"https://github.com/uwiger/gproc",""}},
+{gproc,".*",{git,"https://github.com/uwiger/gproc",""}},{lager,".*",{git,"https://github.com/basho/lager","master"}},{esockd,".*",{git,"https://github.com/emqtt/esockd","master"}},{mochiweb,".*",{git,"https://github.com/emqtt/mochiweb",""}}
-	{lager,".*",{git,"https://github.com/basho/lager","master"}},
-	{esockd,".*",{git,"https://github.com/emqtt/esockd","master"}},
-	{mochiweb,".*",{git,"https://github.com/emqtt/mochiweb",""}}
 ]}.
 {erl_opts, [{parse_transform,lager_transform}]}.

src/emqttd.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,10 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% Facade Module for The EMQ Broker
+%% @doc EMQ Main Module.
 
 -module(emqttd).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -138,23 +140,23 @@ subscriber_down(Subscriber) ->
 
 -spec(hook(atom(), function(), list(any())) -> ok | {error, any()}).
 hook(Hook, Function, InitArgs) ->
-    emqttd_hook:add(Hook, Function, InitArgs).
+    emqttd_hooks:add(Hook, Function, InitArgs).
 
 -spec(hook(atom(), function(), list(any()), integer()) -> ok | {error, any()}).
 hook(Hook, Function, InitArgs, Priority) ->
-    emqttd_hook:add(Hook, Function, InitArgs, Priority).
+    emqttd_hooks:add(Hook, Function, InitArgs, Priority).
 
 -spec(unhook(atom(), function()) -> ok | {error, any()}).
 unhook(Hook, Function) ->
-    emqttd_hook:delete(Hook, Function).
+    emqttd_hooks:delete(Hook, Function).
 
 -spec(run_hooks(atom(), list(any())) -> ok | stop).
 run_hooks(Hook, Args) ->
-    emqttd_hook:run(Hook, Args).
+    emqttd_hooks:run(Hook, Args).
 
 -spec(run_hooks(atom(), list(any()), any()) -> {ok | stop, any()}).
 run_hooks(Hook, Args, Acc) ->
-    emqttd_hook:run(Hook, Args, Acc).
+    emqttd_hooks:run(Hook, Args, Acc).
 
 %%--------------------------------------------------------------------
 %% Debug

src/emqttd_access_control.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,29 +16,25 @@
 
 -module(emqttd_access_control).
 
--include("emqttd.hrl").
-
 -behaviour(gen_server).
 
--define(SERVER, ?MODULE).
+-author("Feng Lee <feng@emqtt.io>").
+
+-include("emqttd.hrl").
 
 %% API Function Exports
--export([start_link/0,
+-export([start_link/0, auth/2, check_acl/3, reload_acl/0, lookup_mods/1,
-         auth/2,       % authentication
+         register_mod/3, register_mod/4, unregister_mod/2, stop/0]).
-         check_acl/3,  % acl check
-         reload_acl/0, % reload acl
-         lookup_mods/1,
-         register_mod/3, register_mod/4,
-         unregister_mod/2,
-         stop/0]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
+-define(SERVER, ?MODULE).
+
 -define(ACCESS_CONTROL_TAB, mqtt_access_control).
 
--type password() :: undefined | binary().
+-type(password() :: undefined | binary()).
 
 -record(state, {}).
 

src/emqttd_access_rule.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,21 +16,24 @@
 
 -module(emqttd_access_rule).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
--type who() :: all | binary() |
+
+-type(who() :: all | binary() |
                {ipaddr, esockd_cidr:cidr_string()} |
                {client, binary()} |
-               {user, binary()}.
+               {user, binary()}).
 
--type access() :: subscribe | publish | pubsub.
+-type(access() :: subscribe | publish | pubsub).
 
--type topic() :: binary().
+-type(topic() :: binary()).
 
--type rule() :: {allow, all} |
+-type(rule() :: {allow, all} |
                 {allow, who(), access(), list(topic())} |
                 {deny, all} |
-                {deny, who(), access(), list(topic())}.
+                {deny, who(), access(), list(topic())}).
 
 -export_type([rule/0]).
 

src/emqttd_acl_internal.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behaviour(emqttd_acl_mod).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -export([all_rules/0]).

src/emqttd_acl_mod.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_acl_mod).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 %%--------------------------------------------------------------------
@@ -26,10 +28,9 @@
 
 -callback(init(AclOpts :: list()) -> {ok, State :: any()}).
 
--callback(check_acl({Client, PubSub, Topic}, State :: any()) -> allow | deny | ignore when
+-callback(check_acl({Client :: mqtt_client(),
-        Client   :: mqtt_client(),
+                     PubSub :: pubsub(),
-        PubSub   :: pubsub(),
+                     Topic  :: binary()}, State :: any()) -> allow | deny | ignore).
-        Topic    :: binary()).
 
 -callback(reload_acl(State :: any()) -> ok | {error, any()}).
 

src/emqttd_alarm.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_alarm).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(gen_event).
 
 -include("emqttd.hrl").
@@ -90,12 +92,12 @@ handle_event({set_alarm, Alarm = #mqtt_alarm{id       = AlarmId,
                               {severity, Severity},
                               {title, iolist_to_binary(Title)},
                               {summary, iolist_to_binary(Summary)},
-                              {ts, emqttd_time:now_to_secs(Timestamp)}]),
+                              {ts, emqttd_time:now_secs(Timestamp)}]),
     emqttd:publish(alarm_msg(alert, AlarmId, Json)),
     {ok, [Alarm#mqtt_alarm{timestamp = Timestamp} | Alarms]};
 
 handle_event({clear_alarm, AlarmId}, Alarms) ->
-    Json = mochijson2:encode([{id, AlarmId}, {ts, emqttd_time:now_to_secs()}]),
+    Json = mochijson2:encode([{id, AlarmId}, {ts, emqttd_time:now_secs()}]),
     emqttd:publish(alarm_msg(clear, AlarmId, Json)),
     {ok, lists:keydelete(AlarmId, 2, Alarms), hibernate};
 

src/emqttd_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,29 +18,26 @@
 
 -behaviour(application).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd_cli.hrl").
 
+-include("emqttd_protocol.hrl").
+
 %% Application callbacks
 -export([start/2, stop/1]).
 
 -export([start_listener/1, stop_listener/1]).
 
-%% MQTT SockOpts
+-type(listener() :: {atom(), esockd:listen_on(), [esockd:option()]}).
--define(MQTT_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true},
-                        {backlog, 512}, {nodelay, true}]).
 
--type listener() :: {atom(), esockd:listen_on(), [esockd:option()]}.
+-define(APP, emqttd).
 
 %%--------------------------------------------------------------------
 %% Application callbacks
 %%--------------------------------------------------------------------
 
--spec(start(StartType, StartArgs) -> {ok, pid()} | {ok, pid(), State} | {error, Reason} when
+start(_Type, _Args) ->
-      StartType :: normal | {takeover, node()} | {failover, node()},
-      StartArgs :: term(),
-      State     :: term(),
-      Reason    :: term()).
-start(_StartType, _StartArgs) ->
     print_banner(),
     emqttd_mnesia:start(),
     {ok, Sup} = emqttd_sup:start_link(),
@@ -63,12 +60,11 @@ stop(_State) ->
 %%--------------------------------------------------------------------
 
 print_banner() ->
-    ?PRINT("starting emqttd on node '~s'~n", [node()]).
+    ?PRINT("starting ~s on node '~s'~n", [?APP, node()]).
 
 print_vsn() ->
     {ok, Vsn} = application:get_key(vsn),
-    {ok, Desc} = application:get_key(description),
+    ?PRINT("~s ~s is running now~n", [?APP, Vsn]).
-    ?PRINT("~s ~s is running now~n", [Desc, Vsn]).
 
 %%--------------------------------------------------------------------
 %% Start Servers
@@ -76,7 +72,7 @@ print_vsn() ->
 
 start_servers(Sup) ->
     Servers = [{"emqttd ctl", emqttd_ctl},
-               {"emqttd hook", emqttd_hook},
+               {"emqttd hook", emqttd_hooks},
                {"emqttd router", emqttd_router},
                {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}},
                {"emqttd stats", emqttd_stats},
@@ -176,14 +172,14 @@ start_listener({Proto, ListenOn, Opts}) when Proto == https; Proto == wss ->
     mochiweb:start_http('mqtt:wss', ListenOn, Opts, {emqttd_http, handle_request, []}).
 
 start_listener(Proto, ListenOn, Opts) ->
-    {ok, Env} = emqttd:env(protocol),
+    Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])),
     MFArgs = {emqttd_client, start_link, [Env]},
     {ok, _} = esockd:open(Proto, ListenOn, merge_sockopts(Opts), MFArgs).
 
 merge_sockopts(Options) ->
-    SockOpts = emqttd_opts:merge(?MQTT_SOCKOPTS,
+    SockOpts = emqttd_misc:merge_opts(
-                                 proplists:get_value(sockopts, Options, [])),
+                 ?MQTT_SOCKOPTS, proplists:get_value(sockopts, Options, [])),
-    emqttd_opts:merge(Options, [{sockopts, SockOpts}]).
+    emqttd_misc:merge_opts(Options, [{sockopts, SockOpts}]).
 
 %%--------------------------------------------------------------------
 %% Stop Listeners
@@ -211,3 +207,4 @@ merge_sockopts_test_() ->
     ?_assert(merge_sockopts(Opts) == [{sockopts, ?MQTT_SOCKOPTS} | Opts]).
 
 -endif.
+

src/emqttd_auth_mod.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_auth_mod).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -export([passwd_hash/2]).
@@ -30,10 +32,10 @@
 
 -callback(init(AuthOpts :: list()) -> {ok, State :: any()}).
 
--callback(check(Client, Password, State) -> ok | | {ok, boolean()} | ignore | {error, string()} when
+-callback(check(Client   :: mqtt_client(),
-        Client   :: mqtt_client(),
+                Password :: binary(),
-        Password :: binary(),
+                State    :: any())
-        State    :: any()).
+          -> ok | | {ok, boolean()} | ignore | {error, string()}).
 
 -callback(description() -> string()).
 

src/emqttd_base62.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2016-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_base62).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -export([encode/1, decode/1]).
 
 %% @doc Encode an integer to base62 string

src/emqttd_boot.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_boot).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -export([apply_module_attributes/1, all_module_attributes/1]).
 
 %% only {F, Args}...

src/emqttd_bridge.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behaviour(gen_server2).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -43,11 +45,11 @@
                 ping_down_interval = ?PING_DOWN_INTERVAL,
                 status             = up}).
 
--type option()  :: {qos, mqtt_qos()} |
+-type(option()  :: {qos, mqtt_qos()} |
                    {topic_suffix, binary()} |
                    {topic_prefix, binary()} |
                    {max_queue_len, pos_integer()} |
-                   {ping_down_interval, pos_integer()}.
+                   {ping_down_interval, pos_integer()}).
 
 -export_type([option/0]).
 

src/emqttd_bridge_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

src/emqttd_bridge_sup_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behavior(supervisor).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -export([start_link/0, bridges/0, start_bridge/2, start_bridge/3, stop_bridge/2]).
 
 -export([init/1]).
@@ -47,7 +49,7 @@ start_bridge(Node, _Topic, _Options) when Node =:= node() ->
     {error, bridge_to_self};
 start_bridge(Node, Topic, Options) when is_atom(Node) andalso is_binary(Topic) ->
     {ok, BridgeEnv} = emqttd:env(bridge),
-    Options1 = emqttd_opts:merge(BridgeEnv, Options),
+    Options1 = emqttd_misc:merge_opts(BridgeEnv, Options),
     supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)).
 
 %% @doc Stop a bridge

src/emqttd_broker.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_internal.hrl").
@@ -40,16 +42,18 @@
 
 -record(state, {started_at, sys_interval, heartbeat, tick_tref, version, sysdescr}).
 
+-define(APP, emqttd).
+
 -define(SERVER, ?MODULE).
 
 -define(BROKER_TAB, mqtt_broker).
 
 %% $SYS Topics of Broker
 -define(SYSTOP_BROKERS, [
-    version,      % Broker version
+    version,  % Broker version
-    uptime,       % Broker uptime
+    uptime,   % Broker uptime
-    datetime,     % Broker local datetime
+    datetime, % Broker local datetime
-    sysdescr      % Broker description
+    sysdescr  % Broker description
 ]).
 
 %%--------------------------------------------------------------------
@@ -74,12 +78,12 @@ notify(EventType, Event) ->
 %% @doc Get broker version
 -spec(version() -> string()).
 version() ->
-    {ok, Version} = application:get_key(emqttd, vsn), Version.
+    {ok, Version} = application:get_key(?APP, vsn), Version.
 
 %% @doc Get broker description
 -spec(sysdescr() -> string()).
 sysdescr() ->
-    {ok, Descr} = application:get_key(emqttd, description), Descr.
+    {ok, Descr} = application:get_key(?APP, description), Descr.
 
 %% @doc Get broker uptime
 -spec(uptime() -> string()).
@@ -161,13 +165,11 @@ retain(brokers) ->
     Payload = list_to_binary(string:join([atom_to_list(N) ||
                     N <- emqttd_mnesia:running_nodes()], ",")),
     Msg = emqttd_message:make(broker, <<"$SYS/brokers">>, Payload),
-    Msg1 = emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg)),
+    emqttd:publish(emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg))).
-    emqttd:publish(Msg1).
 
 retain(Topic, Payload) when is_binary(Payload) ->
     Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload),
-    Msg1 = emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg)),
+    emqttd:publish(emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg))).
-    emqttd:publish(Msg1).
 
 publish(Topic, Payload) when is_binary(Payload) ->
     Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload),

src/emqttd_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_cli).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_cli.hrl").
@@ -57,6 +59,7 @@ is_cmd(Fun) ->
 
 %%--------------------------------------------------------------------
 %% @doc Node status
+
 status([]) ->
     {InternalStatus, _ProvidedStatus} = init:get_status(),
     ?PRINT("Node ~p is ~p~n", [node(), InternalStatus]),
@@ -71,6 +74,7 @@ status(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Query broker
+
 broker([]) ->
     Funs = [sysdescr, version, uptime, datetime],
     foreach(fun(Fun) ->
@@ -105,6 +109,7 @@ broker(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Cluster with other nodes
+
 cluster(["join", SNode]) ->
     case emqttd_cluster:join(emqttd_node:parse_name(SNode)) of
         ok ->
@@ -143,10 +148,12 @@ cluster(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Users usage
+
 users(Args) -> emq_auth_username:cli(Args).
 
 %%--------------------------------------------------------------------
 %% @doc Query clients
+
 clients(["list"]) ->
     dump(mqtt_client);
 
@@ -169,14 +176,17 @@ if_client(ClientId, Fun) ->
 
 %%--------------------------------------------------------------------
 %% @doc Sessions Command
+
 sessions(["list"]) ->
     dump(mqtt_local_session);
 
 %% performance issue?
+
 sessions(["list", "persistent"]) ->
     lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', '_', false, '_'}));
 
 %% performance issue?
+
 sessions(["list", "transient"]) ->
     lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', '_', true, '_'}));
 
@@ -194,6 +204,7 @@ sessions(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Routes Command
+
 routes(["list"]) ->
     Routes = emqttd_router:dump(),
     foreach(fun print/1, Routes);
@@ -207,6 +218,7 @@ routes(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Topics Command
+
 topics(["list"]) ->
     lists:foreach(fun(Topic) -> ?PRINT("~s~n", [Topic]) end, emqttd:topics());
 
@@ -260,14 +272,14 @@ subscriptions(_) ->
             {"subscriptions del <ClientId>",               "Delete static subscriptions manually"},
             {"subscriptions del <ClientId> <Topic>",       "Delete a static subscription manually"}]).
 
-if_could_print(Tab, Fun) ->
+% if_could_print(Tab, Fun) ->
-    case mnesia:table_info(Tab, size) of
+%    case mnesia:table_info(Tab, size) of
-        Size when Size >= ?MAX_LIMIT ->
+%        Size when Size >= ?MAX_LIMIT ->
-            ?PRINT("Could not list, too many ~ss: ~p~n", [Tab, Size]);
+%            ?PRINT("Could not list, too many ~ss: ~p~n", [Tab, Size]);
-        _Size ->
+%        _Size ->
-            Keys = mnesia:dirty_all_keys(Tab),
+%            Keys = mnesia:dirty_all_keys(Tab),
-            foreach(fun(Key) -> Fun(ets:lookup(Tab, Key)) end, Keys)
+%            foreach(fun(Key) -> Fun(ets:lookup(Tab, Key)) end, Keys)
-    end.
+%    end.
 
 if_valid_qos(QoS, Fun) ->
     try list_to_integer(QoS) of
@@ -303,6 +315,7 @@ plugins(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Bridges command
+
 bridges(["list"]) ->
     foreach(fun({Node, Topic, _Pid}) ->
                 ?PRINT("bridge: ~s--~s-->~s~n", [node(), Topic, Node])
@@ -360,6 +373,7 @@ parse_opt(_Cmd, Opt, _Val) ->
 
 %%--------------------------------------------------------------------
 %% @doc vm command
+
 vm([]) ->
     vm(["all"]);
 
@@ -398,6 +412,7 @@ vm(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc mnesia Command
+
 mnesia([]) ->
     mnesia:system_info();
 
@@ -406,6 +421,7 @@ mnesia(_) ->
 
 %%--------------------------------------------------------------------
 %% @doc Trace Command
+
 trace(["list"]) ->
     foreach(fun({{Who, Name}, LogFile}) ->
                 ?PRINT("trace ~s ~s -> ~s~n", [Who, Name, LogFile])
@@ -448,6 +464,7 @@ trace_off(Who, Name) ->
 
 %%--------------------------------------------------------------------
 %% @doc Listeners Command
+
 listeners([]) ->
     foreach(fun({{Protocol, ListenOn}, Pid}) ->
                 Info = [{acceptors,      esockd:get_acceptors(Pid)},
@@ -503,7 +520,7 @@ print(#mqtt_client{client_id = ClientId, clean_sess = CleanSess, username = User
                    peername = Peername, connected_at = ConnectedAt}) ->
     ?PRINT("Client(~s, clean_sess=~s, username=~s, peername=~s, connected_at=~p)~n",
            [ClientId, CleanSess, Username, emqttd_net:format(Peername),
-            emqttd_time:now_to_secs(ConnectedAt)]);
+            emqttd_time:now_secs(ConnectedAt)]);
 
 %% print(#mqtt_topic{topic = Topic, flags = Flags}) ->
 %%    ?PRINT("~s: ~s~n", [Topic, string:join([atom_to_list(F) || F <- Flags], ",")]);
@@ -542,7 +559,7 @@ print(subscription, {Sub, Topic}) ->
     ?PRINT("~s -> ~s~n", [Sub, Topic]).
 
 format(created_at, Val) ->
-    emqttd_time:now_to_secs(Val);
+    emqttd_time:now_secs(Val);
 
 format(_, Val) ->
     Val.

src/emqttd_client.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,24 +14,36 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc MQTT Client Connection
+%% @doc MQTT/TCP Connection
+
 -module(emqttd_client).
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
 
 -include("emqttd_internal.hrl").
 
+-import(proplists, [get_value/2, get_value/3]).
+
 %% API Function Exports
--export([start_link/2, session/1, info/1, kick/1,
+-export([start_link/2]).
-         set_rate_limit/2, get_rate_limit/1]).
+
+%% Management and Monitor API
+-export([info/1, stats/1, kick/1]).
+
+-export([set_rate_limit/2, get_rate_limit/1]).
 
 %% SUB/UNSUB Asynchronously. Called by plugins.
 -export([subscribe/2, unsubscribe/2]).
 
+%% Get the session proc?
+-export([session/1]).
+
 %% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          code_change/3, terminate/2]).
@@ -39,24 +51,25 @@
 %% Client State
 -record(client_state, {connection, connname, peername, peerhost, peerport,
                        await_recv, conn_state, rate_limit, parser_fun,
-                       proto_state, packet_opts, keepalive, mountpoint}).
+                       proto_state, packet_opts, keepalive, enable_stats,
+                       stats_timer}).
 
--define(INFO_KEYS, [peername, peerhost, peerport, await_recv, conn_state]).
+-define(INFO_KEYS, [connname, peername, peerhost, peerport, await_recv, conn_state]).
 
--define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]).
+-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
 
 -define(LOG(Level, Format, Args, State),
             lager:Level("Client(~s): " ++ Format, [State#client_state.connname | Args])).
 
-start_link(Connection, MqttEnv) ->
+start_link(Conn, Env) ->
-    {ok, proc_lib:spawn_link(?MODULE, init, [[Connection, MqttEnv]])}.
+    {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]])}.
-
-session(CPid) ->
-    gen_server:call(CPid, session, infinity).
 
 info(CPid) ->
     gen_server:call(CPid, info, infinity).
 
+stats(CPid) ->
+    gen_server:call(CPid, stats).
+
 kick(CPid) ->
     gen_server:call(CPid, kick).
 
@@ -67,42 +80,49 @@ get_rate_limit(Cpid) ->
     gen_server:call(Cpid, get_rate_limit).
 
 subscribe(CPid, TopicTable) ->
-    gen_server:cast(CPid, {subscribe, TopicTable}).
+    CPid ! {subscribe, TopicTable}.
 
 unsubscribe(CPid, Topics) ->
-    gen_server:cast(CPid, {unsubscribe, Topics}).
+    CPid ! {unsubscribe, Topics}.
 
-init([OriginConn, MqttEnv]) ->
+session(CPid) ->
-    {ok, Connection} = OriginConn:wait(),
+    gen_server2:call(CPid, session, infinity).
+
+%%--------------------------------------------------------------------
+%% gen_server Callbacks
+%%--------------------------------------------------------------------
+
+init([Conn0, Env]) ->
+    {ok, Conn} = Conn0:wait(),
     {PeerHost, PeerPort, PeerName} =
-    case Connection:peername() of
+    case Conn:peername() of
         {ok, Peer = {Host, Port}} ->
             {Host, Port, Peer};
         {error, enotconn} ->
-            Connection:fast_close(),
+            Conn:fast_close(),
             exit(normal);
         {error, Reason} ->
-            Connection:fast_close(),
+            Conn:fast_close(),
             exit({shutdown, Reason})
     end,
     ConnName = esockd_net:format(PeerName),
     Self = self(),
-
     %% Send Packet...
     SendFun = fun(Packet) ->
         Data = emqttd_serializer:serialize(Packet),
         ?LOG(debug, "SEND ~p", [Data], #client_state{connname = ConnName}),
         emqttd_metrics:inc('bytes/sent', iolist_size(Data)),
-        try Connection:async_send(Data) of
+        try Conn:async_send(Data) of
             true -> ok
         catch
             error:Error -> Self ! {shutdown, Error}
         end
     end,
-    ParserFun = emqttd_parser:new(MqttEnv),
+    ParserFun = emqttd_parser:new(Env),
-    ProtoState = emqttd_protocol:init(PeerName, SendFun, MqttEnv),
+    ProtoState = emqttd_protocol:init(PeerName, SendFun, Env),
-    RateLimit = proplists:get_value(rate_limit, Connection:opts()),
+    RateLimit = get_value(rate_limit, Conn:opts()),
-    State = run_socket(#client_state{connection   = Connection,
+    EnableStats = get_value(client_enable_stats, Env, false),
+    State = run_socket(#client_state{connection   = Conn,
                                      connname     = ConnName,
                                      peername     = PeerName,
                                      peerhost     = PeerHost,
@@ -112,20 +132,21 @@ init([OriginConn, MqttEnv]) ->
                                      rate_limit   = RateLimit,
                                      parser_fun   = ParserFun,
                                      proto_state  = ProtoState,
-                                     packet_opts  = MqttEnv}),
+                                     packet_opts  = Env,
-    IdleTimout = proplists:get_value(client_idle_timeout, MqttEnv, 30),
+                                     enable_stats = EnableStats}),
-    gen_server:enter_loop(?MODULE, [], State, timer:seconds(IdleTimout)).
+    IdleTimout = get_value(client_idle_timeout, Env, 30000),
+    gen_server:enter_loop(?MODULE, [], maybe_enable_stats(State), IdleTimout).
 
-handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> 
+handle_call(info, From, State = #client_state{proto_state = ProtoState}) ->
-    {reply, emqttd_protocol:session(ProtoState), State};
-
-handle_call(info, _From, State = #client_state{connection  = Connection,
-                                               proto_state = ProtoState}) ->
-    ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS),
     ProtoInfo  = emqttd_protocol:info(ProtoState),
-    {ok, SockStats} = Connection:getstat(?SOCK_STATS),
+    ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS),
-    {reply, lists:append([ClientInfo, [{proto_info, ProtoInfo},
+    {reply, Stats, _} = handle_call(stats, From, State),
-                                       {sock_stats, SockStats}]]), State};
+    {reply, lists:append([ClientInfo, ProtoInfo, Stats]), State};
+
+handle_call(stats, _From, State = #client_state{proto_state = ProtoState}) ->
+    {reply, lists:append([emqttd_misc:proc_stats(),
+                          emqttd_protocol:stats(ProtoState),
+                          sock_stats(State)]), State};
 
 handle_call(kick, _From, State) ->
     {stop, {shutdown, kick}, ok, State};
@@ -136,45 +157,56 @@ handle_call({set_rate_limit, Rl}, _From, State) ->
 handle_call(get_rate_limit, _From, State = #client_state{rate_limit = Rl}) ->
     {reply, Rl, State};
 
+handle_call(session, _From, State = #client_state{proto_state = ProtoState}) ->
+    {reply, emqttd_protocol:session(ProtoState), State};
+
 handle_call(Req, _From, State) ->
     ?UNEXPECTED_REQ(Req, State).
 
-handle_cast({subscribe, TopicTable}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                emqttd_protocol:handle({subscribe, TopicTable}, ProtoState)
-        end, State);
-
-handle_cast({unsubscribe, Topics}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                emqttd_protocol:handle({unsubscribe, Topics}, ProtoState)
-        end, State);
-
 handle_cast(Msg, State) ->
     ?UNEXPECTED_MSG(Msg, State).
 
-handle_info(timeout, State) ->
+handle_info({subscribe, TopicTable}, State) ->
-    shutdown(idle_timeout, State);
+    with_proto(
+      fun(ProtoState) ->
+          emqttd_protocol:subscribe(TopicTable, ProtoState)
+      end, State);
 
-%% fix issue #535
+handle_info({unsubscribe, Topics}, State) ->
-handle_info({shutdown, Error}, State) ->
+    with_proto(
-    shutdown(Error, State);
+      fun(ProtoState) ->
+          emqttd_protocol:unsubscribe(Topics, ProtoState)
+      end, State);
 
 %% Asynchronous SUBACK
 handle_info({suback, PacketId, GrantedQos}, State) ->
-    with_proto_state(fun(ProtoState) ->
+    with_proto(
-                       Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
+      fun(ProtoState) ->
-                       emqttd_protocol:send(Packet, ProtoState)
+          Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
-                     end, State);
+          emqttd_protocol:send(Packet, ProtoState)
+      end, State);
 
 handle_info({deliver, Message}, State) ->
-    with_proto_state(fun(ProtoState) ->
+    with_proto(
-                       emqttd_protocol:send(Message, ProtoState)
+      fun(ProtoState) ->
-                     end, State);
+          emqttd_protocol:send(Message, ProtoState)
+      end, State);
 
 handle_info({redeliver, {?PUBREL, PacketId}}, State) ->
-    with_proto_state(fun(ProtoState) ->
+    with_proto(
-                       emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState)
+      fun(ProtoState) ->
-                     end, State);
+          emqttd_protocol:pubrel(PacketId, ProtoState)
+      end, State);
+
+handle_info(timeout, State) ->
+    shutdown(idle_timeout, State);
+
+handle_info({timeout, _Timer, emit_stats}, State) ->
+    hibernate(maybe_enable_stats(emit_stats(State)));
+
+%% Fix issue #535
+handle_info({shutdown, Error}, State) ->
+    shutdown(Error, State);
 
 handle_info({shutdown, conflict, {ClientId, NewPid}}, State) ->
     ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State),
@@ -193,26 +225,26 @@ handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) ->
     shutdown(Reason, State);
 
 handle_info({inet_reply, _Sock, ok}, State) ->
-    hibernate(State);
+    {noreply, State};
 
 handle_info({inet_reply, _Sock, {error, Reason}}, State) ->
     shutdown(Reason, State);
 
-handle_info({keepalive, start, Interval}, State = #client_state{connection = Connection}) ->
+handle_info({keepalive, start, Interval}, State = #client_state{connection = Conn}) ->
     ?LOG(debug, "Keepalive at the interval of ~p", [Interval], State),
     StatFun = fun() ->
-                case Connection:getstat([recv_oct]) of
+                case Conn:getstat([recv_oct]) of
                     {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct};
                     {error, Error}              -> {error, Error}
                 end
              end,
     KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}),
-    hibernate(State#client_state{keepalive = KeepAlive});
+    {noreply, stats_by_keepalive(State#client_state{keepalive = KeepAlive})};
 
 handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) ->
     case emqttd_keepalive:check(KeepAlive) of
         {ok, KeepAlive1} ->
-            hibernate(State#client_state{keepalive = KeepAlive1});
+            hibernate(emit_stats(State#client_state{keepalive = KeepAlive1}));
         {error, timeout} ->
             ?LOG(debug, "Keepalive timeout", [], State),
             shutdown(keepalive_timeout, State);
@@ -224,10 +256,10 @@ handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) ->
 handle_info(Info, State) ->
     ?UNEXPECTED_INFO(Info, State).
 
-terminate(Reason, #client_state{connection  = Connection,
+terminate(Reason, #client_state{connection  = Conn,
                                 keepalive   = KeepAlive,
                                 proto_state = ProtoState}) ->
-    Connection:fast_close(),
+    Conn:fast_close(),
     emqttd_keepalive:cancel(KeepAlive),
     case {ProtoState, Reason} of
         {undefined, _} ->
@@ -245,10 +277,6 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-with_proto_state(Fun, State = #client_state{proto_state = ProtoState}) ->
-    {ok, ProtoState1} = Fun(ProtoState),
-    hibernate(State#client_state{proto_state = ProtoState1}).
-
 %% Receive and parse tcp data
 received(<<>>, State) ->
     hibernate(State);
@@ -258,7 +286,7 @@ received(Bytes, State = #client_state{parser_fun  = ParserFun,
                                       proto_state = ProtoState}) ->
     case catch ParserFun(Bytes) of
         {more, NewParser}  ->
-            noreply(run_socket(State#client_state{parser_fun = NewParser}));
+            {noreply, run_socket(State#client_state{parser_fun = NewParser})};
         {ok, Packet, Rest} ->
             emqttd_metrics:received(Packet),
             case emqttd_protocol:received(Packet, ProtoState) of
@@ -289,7 +317,7 @@ rate_limit(Size, State = #client_state{rate_limit = Rl}) ->
         {0, Rl1} ->
             run_socket(State#client_state{conn_state = running, rate_limit = Rl1});
         {Pause, Rl1} ->
-            ?LOG(error, "Rate limiter pause for ~p", [Pause], State),
+            ?LOG(warning, "Rate limiter pause for ~p", [Pause], State),
             erlang:send_after(Pause, self(), activate_sock),
             State#client_state{conn_state = blocked, rate_limit = Rl1}
     end.
@@ -298,12 +326,36 @@ run_socket(State = #client_state{conn_state = blocked}) ->
     State;
 run_socket(State = #client_state{await_recv = true}) ->
     State;
-run_socket(State = #client_state{connection = Connection}) ->
+run_socket(State = #client_state{connection = Conn}) ->
-    Connection:async_recv(0, infinity),
+    Conn:async_recv(0, infinity),
     State#client_state{await_recv = true}.
 
-noreply(State) ->
+with_proto(Fun, State = #client_state{proto_state = ProtoState}) ->
-    {noreply, State}.
+    {ok, ProtoState1} = Fun(ProtoState),
+    {noreply, State#client_state{proto_state = ProtoState1}}.
+
+maybe_enable_stats(State = #client_state{enable_stats = false}) ->
+    State;
+maybe_enable_stats(State = #client_state{enable_stats = keepalive}) ->
+    State;
+maybe_enable_stats(State = #client_state{enable_stats = Interval}) ->
+    State#client_state{stats_timer = emqttd_misc:start_timer(Interval, self(), emit_stats)}.
+
+stats_by_keepalive(State) ->
+    State#client_state{enable_stats = keepalive}.
+
+emit_stats(State = #client_state{enable_stats = false}) ->
+    State;
+emit_stats(State = #client_state{proto_state = ProtoState}) ->
+    {reply, Stats, _} = handle_call(stats, undefined, State),
+    emqttd_stats:set_client_stats(emqttd_protocol:clientid(ProtoState), Stats),
+    State.
+
+sock_stats(#client_state{connection = Conn}) ->
+    case Conn:getstat(?SOCK_STATS) of
+        {ok,   Ss} -> Ss;
+        {error, _} -> []
+    end.
 
 hibernate(State) ->
     {noreply, State, hibernate}.

src/emqttd_cluster.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_cluster).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 %% Cluster API

src/emqttd_cm.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,10 +15,13 @@
 %%--------------------------------------------------------------------
 
 %% @doc MQTT Client Manager
+
 -module(emqttd_cm).
 
 -behaviour(gen_server2).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_internal.hrl").
@@ -120,6 +123,7 @@ handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) ->
         {ok, {ClientId, DownPid}} ->
             case lookup_proc(ClientId) of
                 DownPid ->
+                    emqttd_stats:del_client_stats(ClientId),
                     ets:delete(mqtt_client, ClientId);
                 _ ->
                     ignore

src/emqttd_cm_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,10 +15,13 @@
 %%--------------------------------------------------------------------
 
 %% @doc Client Manager Supervisor.
+
 -module(emqttd_cm_sup).
 
 -behaviour(supervisor).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 %% API

src/emqttd_ctl.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_cli.hrl").
@@ -134,7 +136,9 @@ next_seq(State = #state{seq = Seq}) ->
     State#state{seq = Seq + 1}.
 
 -ifdef(TEST).
+
 -include_lib("eunit/include/eunit.hrl").
+
 register_cmd_test_() ->
     {setup, 
         fun() ->
@@ -149,4 +153,5 @@ register_cmd_test_() ->
                 [?_assertMatch([{{0,test0},{?MODULE, test0}, []}], ets:lookup(?CMD_TAB, {Seq,test0}))]
         end
     }.
+
 -endif.

src/emqttd_gen_mod.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,15 +15,18 @@
 %%--------------------------------------------------------------------
 
 %% @doc emqttd gen_mod behaviour
+
 -module(emqttd_gen_mod).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -ifdef(use_specs).
 
--callback load(Opts :: any()) -> ok | {error, any()}.
+-callback(load(Opts :: any()) -> ok | {error, any()}).
 
--callback unload(State :: any()) -> any().
+-callback(unload(State :: any()) -> any()).
 
 -else.
 
@@ -35,4 +38,3 @@ behaviour_info(_Other) ->
     undefined.
 
 -endif.
-

src/emqttd_guid.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -27,13 +27,14 @@
 %% 4. Sequence:  2 bytes sequence in one process
 %%
 %% @end
+
 -module(emqttd_guid).
 
 -export([gen/0, new/0, timestamp/1, to_hexstr/1, from_hexstr/1, to_base62/1, from_base62/1]).
 
 -define(MAX_SEQ, 16#FFFF).
 
--type guid() :: <<_:128>>.
+-type(guid() :: <<_:128>>).
 
 %% @doc Generate a global unique id.
 -spec(gen() -> guid()).

src/emqttd_hooks.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2016 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,12 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqttd_hook).
+-module(emqttd_hooks).
-
--author("Feng Lee <feng@emqtt.io>").
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 %% Start
 -export([start_link/0]).
 
@@ -40,10 +40,6 @@
 
 -define(HOOK_TAB, mqtt_hook).
 
-%%--------------------------------------------------------------------
-%% Start API
-%%--------------------------------------------------------------------
-
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 

src/emqttd_http.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,9 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd http publish API and websocket client.
+%% @doc HTTP publish API and websocket client.
+
 -module(emqttd_http).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").

src/emqttd_inflight.erl

@@ -0,0 +1,94 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc Inflight Window that wraps the gb_trees.
+
+-module(emqttd_inflight).
+
+-author("Feng Lee <feng@emqtt.io>").
+
+-export([new/1, contain/2, lookup/2, insert/3, update/3, delete/2, values/1,
+         to_list/1, size/1, max_size/1, is_full/1, is_empty/1, window/1]).
+
+-type(inflight() :: {?MODULE, list()}).
+
+-export_type([inflight/0]).
+
+-spec(new(non_neg_integer()) -> inflight()).
+new(MaxSize) when MaxSize >= 0 ->
+    {?MODULE, [MaxSize, gb_trees:empty()]}.
+
+-spec(contain(Key :: any(), inflight()) -> boolean()).
+contain(Key, {?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:is_defined(Key, Tree).
+
+-spec(lookup(Key :: any(), inflight()) -> any()).
+lookup(Key, {?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:get(Key, Tree).
+
+-spec(insert(Key :: any(), Value :: any(), inflight()) -> inflight()).
+insert(Key, Value, {?MODULE, [MaxSize, Tree]}) ->
+    {?MODULE, [MaxSize, gb_trees:insert(Key, Value, Tree)]}.
+
+-spec(delete(Key :: any(), inflight()) -> inflight()).
+delete(Key, {?MODULE, [MaxSize, Tree]}) ->
+    {?MODULE, [MaxSize, gb_trees:delete(Key, Tree)]}.
+
+-spec(update(Key :: any(), Val :: any(), inflight()) -> inflight()).
+update(Key, Val, {?MODULE, [MaxSize, Tree]}) ->
+    {?MODULE, [MaxSize, gb_trees:update(Key, Val, Tree)]}.
+
+-spec(is_full(inflight()) -> boolean()).
+is_full({?MODULE, [0, _Tree]}) ->
+    false;
+is_full({?MODULE, [MaxSize, Tree]}) ->
+    MaxSize =< gb_trees:size(Tree).
+
+-spec(is_empty(inflight()) -> boolean()).
+is_empty({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:is_empty(Tree).
+
+-spec(smallest(inflight()) -> {K :: any(), V :: any()}).
+smallest({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:smallest(Tree).
+
+-spec(largest(inflight()) -> {K :: any(), V :: any()}).
+largest({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:largest(Tree).
+
+-spec(values(inflight()) -> list()).
+values({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:values(Tree).
+
+-spec(to_list(inflight()) -> list({K :: any(), V :: any()})).
+to_list({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:to_list(Tree).
+
+-spec(window(inflight()) -> list()).
+window(Inflight = {?MODULE, [_MaxSize, Tree]}) ->
+    case gb_trees:is_empty(Tree) of
+        true  -> [];
+        false -> [Key || {Key, _Val} <- [smallest(Inflight), largest(Inflight)]]
+    end.
+
+-spec(size(inflight()) -> non_neg_integer()).
+size({?MODULE, [_MaxSize, Tree]}) ->
+    gb_trees:size(Tree).
+
+-spec(max_size(inflight()) -> non_neg_integer()).
+max_size({?MODULE, [MaxSize, _Tree]}) ->
+    MaxSize.
+

src/emqttd_keepalive.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,15 +15,18 @@
 %%--------------------------------------------------------------------
 
 %% @doc Client Keepalive
+
 -module(emqttd_keepalive).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -export([start/3, check/1, cancel/1]).
 
--record(keepalive, {statfun, statval,
+-record(keepalive, {statfun, statval, tsec, tmsg, tref, repeat = 0}).
-                    tsec, tmsg, tref,
-                    repeat = 0}).
 
--type keepalive() :: #keepalive{}.
+-type(keepalive() :: #keepalive{}).
+
+-export_type([keepalive/0]).
 
 %% @doc Start a keepalive
 -spec(start(fun(), integer(), any()) -> undefined | keepalive()).

src/emqttd_message.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,8 +15,11 @@
 %%--------------------------------------------------------------------
 
 %% @doc MQTT Message Functions
+
 -module(emqttd_message).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -42,7 +45,7 @@ make(From, Qos, Topic, Payload) ->
                   qos       = ?QOS_I(Qos),
                   topic     = Topic,
                   payload   = Payload,
-                  timestamp = emqttd_time:now_to_secs()}.
+                  timestamp = os:timestamp()}.
 
 %% @doc Message from Packet
 -spec(from_packet(mqtt_packet()) -> mqtt_message()).
@@ -60,7 +63,7 @@ from_packet(#mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
                   dup       = Dup,
                   topic     = Topic,
                   payload   = Payload,
-                  timestamp = emqttd_time:now_to_secs()};
+                  timestamp = os:timestamp()};
 
 from_packet(#mqtt_packet_connect{will_flag  = false}) ->
     undefined;
@@ -78,7 +81,7 @@ from_packet(#mqtt_packet_connect{client_id   = ClientId,
                   qos       = Qos,
                   dup       = false,
                   payload   = Msg, 
-                  timestamp = emqttd_time:now_to_secs()}.
+                  timestamp = os:timestamp()}.
 
 from_packet(ClientId, Packet) ->
     Msg = from_packet(Packet),

src/emqttd_metrics.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,11 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd metrics. responsible for collecting broker metrics.
 -module(emqttd_metrics).
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -37,7 +38,7 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
--record(state, {tick_tref}).
+-record(state, {tick}).
 
 -define(METRIC_TAB, mqtt_metric).
 
@@ -57,12 +58,16 @@
     {counter, 'packets/publish/sent'},     % PUBLISH packets sent
     {counter, 'packets/puback/received'},  % PUBACK packets received
     {counter, 'packets/puback/sent'},      % PUBACK packets sent
+    {counter, 'packets/puback/missed'},    % PUBACK packets missed
     {counter, 'packets/pubrec/received'},  % PUBREC packets received
     {counter, 'packets/pubrec/sent'},      % PUBREC packets sent
+    {counter, 'packets/pubrec/missed'},    % PUBREC packets missed
     {counter, 'packets/pubrel/received'},  % PUBREL packets received
     {counter, 'packets/pubrel/sent'},      % PUBREL packets sent
+    {counter, 'packets/pubrel/missed'},    % PUBREL packets missed
     {counter, 'packets/pubcomp/received'}, % PUBCOMP packets received
     {counter, 'packets/pubcomp/sent'},     % PUBCOMP packets sent
+    {counter, 'packets/pubcomp/missed'},   % PUBCOMP packets missed
     {counter, 'packets/subscribe'},        % SUBSCRIBE Packets received 
     {counter, 'packets/suback'},           % SUBACK packets sent 
     {counter, 'packets/unsubscribe'},      % UNSUBSCRIBE Packets received
@@ -74,14 +79,15 @@
 
 %% Messages sent and received of broker
 -define(SYSTOP_MESSAGES, [
-    {counter, 'messages/received'},      % Messages received
+    {counter, 'messages/received'},      % All Messages received
-    {counter, 'messages/sent'},          % Messages sent
+    {counter, 'messages/sent'},          % All Messages sent
-    {counter, 'messages/qos0/received'}, % Messages received
+    {counter, 'messages/qos0/received'}, % QoS0 Messages received
-    {counter, 'messages/qos0/sent'},     % Messages sent
+    {counter, 'messages/qos0/sent'},     % QoS0 Messages sent
-    {counter, 'messages/qos1/received'}, % Messages received
+    {counter, 'messages/qos1/received'}, % QoS1 Messages received
-    {counter, 'messages/qos1/sent'},     % Messages sent
+    {counter, 'messages/qos1/sent'},     % QoS1 Messages sent
-    {counter, 'messages/qos2/received'}, % Messages received
+    {counter, 'messages/qos2/received'}, % QoS2 Messages received
-    {counter, 'messages/qos2/sent'},     % Messages sent
+    {counter, 'messages/qos2/sent'},     % QoS2 Messages sent
+    {counter, 'messages/qos2/dropped'},  % QoS2 Messages dropped
     {gauge,   'messages/retained'},      % Messagea retained
     {counter, 'messages/dropped'}        % Messages dropped
 ]).
@@ -138,7 +144,7 @@ qos_received(?QOS_2) ->
 sent(?PUBLISH_PACKET(_Qos, <<"$SYS/", _/binary>>, _, _)) ->
     ignore;
 sent(Packet) ->
-    emqttd_metrics:inc('packets/sent'),
+    inc('packets/sent'),
     sent1(Packet).
 sent1(?PUBLISH_PACKET(Qos, _PktId)) ->
     inc('packets/publish/sent'),
@@ -245,7 +251,7 @@ init([]) ->
     % $SYS Topics for metrics
     % [ok = emqttd:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics],
     % Tick to publish metrics
-    {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}.
+    {ok, #state{tick = emqttd_broker:start_tick(tick)}, hibernate}.
 
 handle_call(_Req, _From, State) ->
     {reply, error, State}.
@@ -261,7 +267,7 @@ handle_info(tick, State) ->
 handle_info(_Info, State) ->
     {noreply, State}.
 
-terminate(_Reason, #state{tick_tref = TRef}) ->
+terminate(_Reason, #state{tick = TRef}) ->
     emqttd_broker:stop_tick(TRef).
 
 code_change(_OldVsn, State, _Extra) ->

src/emqttd_misc.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,12 +14,15 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqttd_opts).
+-module(emqttd_misc).
 
--export([merge/2]).
+-author("Feng Lee <feng@emqtt.io>").
+
+-export([merge_opts/2, start_timer/2, start_timer/3, cancel_timer/1,
+         proc_stats/0, proc_stats/1]).
 
 %% @doc Merge Options
-merge(Defaults, Options) ->
+merge_opts(Defaults, Options) ->
     lists:foldl(
         fun({Opt, Val}, Acc) ->
                 case lists:keymember(Opt, 1, Acc) of
@@ -33,3 +36,28 @@ merge(Defaults, Options) ->
                 end
         end, Defaults, Options).
 
+-spec(start_timer(integer(), term()) -> reference()).
+start_timer(Interval, Msg) ->
+    start_timer(Interval, self(), Msg).
+
+-spec(start_timer(integer(), pid() | atom(), term()) -> reference()).
+start_timer(Interval, Dest, Msg) ->
+    erlang:start_timer(Interval, Dest, Msg).
+
+-spec(cancel_timer(undefined | reference()) -> ok).
+cancel_timer(undefined) ->
+    ok;
+cancel_timer(Timer) ->
+    case catch erlang:cancel_timer(Timer) of
+        false -> receive {timeout, Timer, _} -> ok after 0 -> ok end;
+        _ -> ok
+    end.
+
+proc_stats() ->
+    proc_stats(self()).
+
+proc_stats(Pid) ->
+    Stats = process_info(Pid, [message_queue_len, heap_size, reductions]),
+    {value, {_, V}, Stats1} = lists:keytake(message_queue_len, 1, Stats),
+    [{mailbox_len, V} | Stats1].
+

src/emqttd_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_mnesia).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_internal.hrl").

src/emqttd_mod_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -39,11 +39,7 @@ start_link() ->
 start_child(ChildSpec) when is_tuple(ChildSpec) ->
     supervisor:start_child(?MODULE, ChildSpec).
 
-%%
+start_child(Mod, Type) when is_atom(Mod) andalso is_atom(Type) ->
-%% start_child(Mod::atom(), Type::type()) -> {ok, pid()}
-%% @type type() = worker | supervisor
-%%
-start_child(Mod, Type) when is_atom(Mod) and is_atom(Type) ->
     supervisor:start_child(?MODULE, ?CHILD(Mod, Type)).
 
 -spec(stop_child(any()) -> ok | {error, any()}).

src/emqttd_mqueue.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -43,35 +43,33 @@
 
 -module(emqttd_mqueue).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
 
 -import(proplists, [get_value/3]).
 
--export([new/3, type/1, name/1, is_empty/1, len/1, max_len/1, in/2, out/1, stats/1]).
+-export([new/3, type/1, name/1, is_empty/1, len/1, max_len/1, in/2, out/1,
+         dropped/1, stats/1]).
 
 -define(LOW_WM, 0.2).
 
 -define(HIGH_WM, 0.6).
 
--type priority() :: {iolist(), pos_integer()}.
+-type(priority() :: {iolist(), pos_integer()}).
 
--type option() :: {type, simple | priority}
+-type(option() :: {type, simple | priority}
                 | {max_length, pos_integer() | infinity}
                 | {priority, list(priority())}
-                | {low_watermark, float()}     %% Low watermark
+                | {low_watermark, float()}  %% Low watermark
-                | {high_watermark, float()}    %% High watermark
+                | {high_watermark, float()} %% High watermark
-                | {queue_qos0, boolean()}.     %% Queue Qos0?
+                | {queue_qos0, boolean()}). %% Queue Qos0?
 
--type mqueue_option() :: {max_length, pos_integer()} %% Max queue length
+-type(stat() :: {max_len, infinity | pos_integer()}
-                       | {low_watermark, float()}    %% Low watermark
-                       | {high_watermark, float()}   %% High watermark
-                       | {queue_qos0, boolean()}.    %% Queue Qos0
-
--type stat() :: {max_len, infinity | pos_integer()}
               | {len, non_neg_integer()}
-              | {dropped, non_neg_integer()}.
+              | {dropped, non_neg_integer()}).
 
 -record(mqueue, {type :: simple | priority,
                  name, q :: queue:queue() | priority_queue:q(),
@@ -83,12 +81,12 @@
                  qos0 = false, dropped = 0,
                  alarm_fun}).
 
--type mqueue() :: #mqueue{}.
+-type(mqueue() :: #mqueue{}).
 
 -export_type([mqueue/0, priority/0, option/0]).
 
 %% @doc New Queue.
--spec(new(iolist(), list(mqueue_option()), fun()) -> mqueue()).
+-spec(new(iolist(), list(option()), fun()) -> mqueue()).
 new(Name, Opts, AlarmFun) ->
     Type = get_value(type, Opts, simple),
     MaxLen = get_value(max_length, Opts, infinity),
@@ -141,6 +139,10 @@ len(#mqueue{type = priority, q = Q})   -> priority_queue:len(Q).
 
 max_len(#mqueue{max_len= MaxLen}) -> MaxLen.
 
+%% @doc Dropped of the mqueue
+-spec(dropped(mqueue()) -> non_neg_integer()).
+dropped(#mqueue{dropped = Dropped}) -> Dropped.
+
 %% @doc Stats of the mqueue
 -spec(stats(mqueue()) -> [stat()]).
 stats(#mqueue{type = Type, q = Q, max_len = MaxLen, len = Len, dropped = Dropped}) ->
@@ -208,14 +210,12 @@ maybe_set_alarm(MQ = #mqueue{name = Name, len = Len, high_wm = HighWM, alarm_fun
                         title = io_lib:format("Queue ~s high-water mark", [Name]),
                         summary = io_lib:format("queue len ~p > high_watermark ~p", [Len, HighWM])},
     MQ#mqueue{alarm_fun = AlarmFun(alert, Alarm)};
-
 maybe_set_alarm(MQ) ->
     MQ.
 
 maybe_clear_alarm(MQ = #mqueue{name = Name, len = Len, low_wm = LowWM, alarm_fun = AlarmFun})
     when Len < LowWM ->
     MQ#mqueue{alarm_fun = AlarmFun(clear, list_to_binary(["queue_high_watermark.", Name]))};
-
 maybe_clear_alarm(MQ) ->
     MQ.
 

src/emqttd_net.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,7 +14,6 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd net utilities.
 -module(emqttd_net).
 
 -include_lib("kernel/include/inet.hrl").

src/emqttd_node.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_node).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -import(lists, [concat/1]).
 
 -export([is_aliving/1, parse_name/1]).

src/emqttd_packet.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,9 +14,10 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc MQTT Packet Functions
 -module(emqttd_packet).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -28,12 +29,13 @@
 
 %% @doc Protocol name of version
 -spec(protocol_name(mqtt_vsn()) -> binary()).
-protocol_name(Ver) when Ver =:= ?MQTT_PROTO_V31; Ver =:= ?MQTT_PROTO_V311 ->
+protocol_name(?MQTT_PROTO_V3) -> <<"MQIsdp">>;
-    proplists:get_value(Ver, ?PROTOCOL_NAMES).
+protocol_name(?MQTT_PROTO_V4) -> <<"MQTT">>;
+protocol_name(?MQTT_PROTO_V5) -> <<"MQTT">>.
 
 %% @doc Name of MQTT packet type
 -spec(type_name(mqtt_packet_type()) -> atom()).
-type_name(Type) when Type > ?RESERVED andalso Type =< ?DISCONNECT ->
+type_name(Type) when Type > ?RESERVED andalso Type =< ?AUTH ->
     lists:nth(Type, ?TYPE_NAMES).
 
 %% @doc Connack Name

src/emqttd_parser.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc MQTT Packet Parser
 -module(emqttd_parser).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -26,9 +28,9 @@
 
 -record(mqtt_packet_limit, {max_packet_size}).
 
--type option() :: {atom(),  any()}.
+-type(option() :: {atom(),  any()}).
 
--type parser() :: fun( (binary()) -> any() ).
+-type(parser() :: fun( (binary()) -> any() )).
 
 %% @doc Initialize a parser
 -spec(new(Opts :: [option()]) -> parser()).
@@ -86,7 +88,7 @@ parse_frame(Bin, #mqtt_packet_header{type = Type, qos  = Qos} = Header, Length)
               WillRetain   : 1,
               WillQos      : 2,
               WillFlag     : 1,
-              CleanSession : 1,
+              CleanSess    : 1,
               _Reserved    : 1,
               KeepAlive    : 16/big,
               Rest3/binary>>   = Rest2,
@@ -104,7 +106,7 @@ parse_frame(Bin, #mqtt_packet_header{type = Type, qos  = Qos} = Header, Length)
                            will_retain = bool(WillRetain),
                            will_qos    = WillQos,
                            will_flag   = bool(WillFlag),
-                           clean_sess  = bool(CleanSession),
+                           clean_sess  = bool(CleanSess),
                            keep_alive  = KeepAlive,
                            client_id   = ClientId,
                            will_topic  = WillTopic,

src/emqttd_plugins.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_plugins).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -export([init/0]).

src/emqttd_pmon.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2016-2017 Feng Lee <feng@emqtt.io>. All Rights Reserved.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,11 +18,14 @@
 
 -author("Feng Lee <feng@emqtt.io>").
 
--type(pmon() :: {?MODULE, map()}).
-
 -export([new/0, monitor/2, demonitor/2, erase/2]).
 
-new() -> {?MODULE, [maps:new()]}.
+-type(pmon() :: {?MODULE, map()}).
+
+-export_type([pmon/0]).
+
+new() ->
+    {?MODULE, [maps:new()]}.
 
 -spec(monitor(pid(), pmon()) -> pmon()).
 monitor(Pid, PM = {?MODULE, [M]}) ->

src/emqttd_pool_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc Common Pool Supervisor
 -module(emqttd_pool_sup).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(supervisor).
 
 %% API

src/emqttd_pooler.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -39,6 +39,7 @@ start_link() ->
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
+
 -spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, any()}).
 start_link(Pool, Id) ->
     gen_server:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id], []).

src/emqttd_protocol.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_protocol).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -25,9 +27,11 @@
 -import(proplists, [get_value/2, get_value/3]).
 
 %% API
--export([init/3, info/1, clientid/1, client/1, session/1]).
+-export([init/3, info/1, stats/1, clientid/1, client/1, session/1]).
 
--export([received/2, handle/2, send/2, redeliver/2, shutdown/2]).
+-export([subscribe/2, unsubscribe/2, pubrel/2, shutdown/2]).
+
+-export([received/2, send/2]).
 
 -export([process/2]).
 
@@ -39,17 +43,20 @@
                       session, ws_initial_headers, %% Headers from first HTTP request for websocket client
                       connected_at}).
 
--type proto_state() :: #proto_state{}.
+-type(proto_state() :: #proto_state{}).
 
 -define(INFO_KEYS, [client_id, username, clean_sess, proto_ver, proto_name,
                     keepalive, will_msg, ws_initial_headers, connected_at]).
 
+-define(STATS_KEYS, [recv_pkt, recv_msg, send_pkt, send_msg]).
+
 -define(LOG(Level, Format, Args, State),
             lager:Level([{client, State#proto_state.client_id}], "Client(~s@~s): " ++ Format,
                         [State#proto_state.client_id, esockd_net:format(State#proto_state.peername) | Args])).
 
 %% @doc Init protocol
 init(Peername, SendFun, Opts) ->
+    lists:foreach(fun(K) -> put(K, 0) end, ?STATS_KEYS),
     MaxLen = get_value(max_clientid_len, Opts, ?MAX_CLIENTID_LEN),
     WsInitialHeaders = get_value(ws_initial_headers, Opts),
     #proto_state{peername           = Peername,
@@ -61,6 +68,9 @@ init(Peername, SendFun, Opts) ->
 info(ProtoState) ->
     ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS).
 
+stats(_ProtoState) ->
+    [{K, get(K)} || K <- ?STATS_KEYS].
+
 clientid(#proto_state{client_id = ClientId}) ->
     ClientId.
 
@@ -115,22 +125,22 @@ received(Packet = ?PACKET(_Type), State) ->
             {error, Reason, State}
     end.
 
-handle({subscribe, RawTopicTable}, ProtoState = #proto_state{client_id = ClientId,
+subscribe(RawTopicTable, ProtoState = #proto_state{client_id = ClientId,
-                                                             username  = Username,
+                                                   username  = Username,
-                                                             session   = Session}) ->
+                                                   session   = Session}) ->
     TopicTable = parse_topic_table(RawTopicTable),
-    case emqttd:run_hooks('client.subscribe', [ClientId, Username], TopicTable) of
+    case emqttd_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of
         {ok, TopicTable1} ->
             emqttd_session:subscribe(Session, TopicTable1);
         {stop, _} ->
             ok
     end,
-    {ok, ProtoState};
+    {ok, ProtoState}.
 
-handle({unsubscribe, RawTopics}, ProtoState = #proto_state{client_id = ClientId,
+unsubscribe(RawTopics, ProtoState = #proto_state{client_id = ClientId,
-                                                           username  = Username,
+                                                 username  = Username,
-                                                           session   = Session}) ->
+                                                 session   = Session}) ->
-    case emqttd:run_hooks('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
+    case emqttd_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
         {ok, TopicTable} ->
             emqttd_session:unsubscribe(Session, TopicTable);
         {stop, _} ->
@@ -138,6 +148,9 @@ handle({unsubscribe, RawTopics}, ProtoState = #proto_state{client_id = ClientId,
     end,
     {ok, ProtoState}.
 
+%% @doc Send PUBREL
+pubrel(PacketId, State) -> send(?PUBREL_PACKET(PacketId), State).
+
 process(Packet = ?CONNECT_PACKET(Var), State0) ->
 
     #mqtt_packet_connect{proto_ver  = ProtoVer,
@@ -187,7 +200,7 @@ process(Packet = ?CONNECT_PACKET(Var), State0) ->
             {ReturnCode, false, State1}
     end,
     %% Run hooks
-    emqttd:run_hooks('client.connected', [ReturnCode1], client(State3)),
+    emqttd_hooks:run('client.connected', [ReturnCode1], client(State3)),
     %% Send connack
     send(?CONNACK_PACKET(ReturnCode1, sp(SessPresent)), State3),
     %% stop if authentication failure
@@ -220,8 +233,11 @@ process(?SUBSCRIBE_PACKET(PacketId, []), State) ->
     send(?SUBACK_PACKET(PacketId, []), State);
 
 %% TODO: refactor later...
-process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable), State = #proto_state{session = Session,
+process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable),
-        client_id = ClientId, username = Username, is_superuser = IsSuperuser}) ->
+        State = #proto_state{session      = Session,
+                             client_id    = ClientId,
+                             username     = Username,
+                             is_superuser = IsSuperuser}) ->
     Client = client(State), TopicTable = parse_topic_table(RawTopicTable),
     AllowDenies = if
                     IsSuperuser -> [];
@@ -232,7 +248,7 @@ process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable), State = #proto_state{session
             ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicTable], State),
             send(?SUBACK_PACKET(PacketId, [16#80 || _ <- TopicTable]), State);
         false ->
-            case emqttd:run_hooks('client.subscribe', [ClientId, Username], TopicTable) of
+            case emqttd_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of
                 {ok, TopicTable1} ->
                     emqttd_session:subscribe(Session, PacketId, TopicTable1), {ok, State};
                 {stop, _} ->
@@ -244,9 +260,11 @@ process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable), State = #proto_state{session
 process(?UNSUBSCRIBE_PACKET(PacketId, []), State) ->
     send(?UNSUBACK_PACKET(PacketId), State);
 
-process(?UNSUBSCRIBE_PACKET(PacketId, RawTopics), State = #proto_state{
+process(?UNSUBSCRIBE_PACKET(PacketId, RawTopics),
-        client_id = ClientId, username = Username, session = Session}) ->
+        State = #proto_state{client_id = ClientId,
-    case emqttd:run_hooks('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
+                             username  = Username,
+                             session   = Session}) ->
+    case emqttd_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of
         {ok, TopicTable} ->
             emqttd_session:unsubscribe(Session, TopicTable);
         {stop, _} ->
@@ -262,7 +280,9 @@ process(?PACKET(?DISCONNECT), State) ->
     {stop, normal, State#proto_state{will_msg = undefined}}.
 
 publish(Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId),
-        #proto_state{client_id = ClientId, username = Username, session = Session}) ->
+        #proto_state{client_id = ClientId,
+                     username  = Username,
+                     session   = Session}) ->
     Msg = emqttd_message:from_packet(Username, ClientId, Packet),
     emqttd_session:publish(Session, Msg);
 
@@ -287,7 +307,7 @@ with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId),
 -spec(send(mqtt_message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}).
 send(Msg, State = #proto_state{client_id = ClientId, username = Username})
         when is_record(Msg, mqtt_message) ->
-    emqttd:run_hooks('message.delivered', [ClientId, Username], Msg),
+    emqttd_hooks:run('message.delivered', [ClientId, Username], Msg),
     send(emqttd_message:to_packet(Msg), State);
 
 send(Packet, State = #proto_state{sendfun = SendFun})
@@ -297,15 +317,15 @@ send(Packet, State = #proto_state{sendfun = SendFun})
     SendFun(Packet),
     {ok, State}.
 
-trace(recv, Packet, ProtoState) ->
+trace(recv, Packet = ?PACKET(Type), ProtoState) ->
+    inc(recv_pkt), ?IF(Type =:= ?PUBLISH, inc(recv_msg), ok),
     ?LOG(info, "RECV ~s", [emqttd_packet:format(Packet)], ProtoState);
 
-trace(send, Packet, ProtoState) ->
+trace(send, Packet = ?PACKET(Type), ProtoState) ->
+    inc(send_pkt), ?IF(Type =:= ?PUBLISH, inc(send_msg), ok),
     ?LOG(info, "SEND ~s", [emqttd_packet:format(Packet)], ProtoState).
 
-%% @doc redeliver PUBREL PacketId
+inc(Key) -> put(Key, get(Key) + 1).
-redeliver({?PUBREL, PacketId}, State) ->
-    send(?PUBREL_PACKET(PacketId), State).
 
 stop_if_auth_failure(RC, State) when RC == ?CONNACK_CREDENTIALS; RC == ?CONNACK_AUTH ->
     {stop, {shutdown, auth_failure}, State};
@@ -325,7 +345,7 @@ shutdown(Error, State = #proto_state{will_msg = WillMsg}) ->
     ?LOG(info, "Shutdown for ~p", [Error], State),
     Client = client(State),
     send_willmsg(Client, WillMsg),
-    emqttd:run_hooks('client.disconnected', [Error], Client),
+    emqttd_hooks:run('client.disconnected', [Error], Client),
     %% let it down
     %% emqttd_cm:unreg(ClientId).
     ok.
@@ -375,19 +395,19 @@ validate_protocol(#mqtt_packet_connect{proto_ver = Ver, proto_name = Name}) ->
 
 validate_clientid(#mqtt_packet_connect{client_id = ClientId},
                   #proto_state{max_clientid_len = MaxLen})
-    when (size(ClientId) >= 1) andalso (size(ClientId) =< MaxLen) ->
+    when (byte_size(ClientId) >= 1) andalso (byte_size(ClientId) =< MaxLen) ->
     true;
 
 %% Issue#599: Null clientId and clean_sess = false
 validate_clientid(#mqtt_packet_connect{client_id  = ClientId,
                                        clean_sess = CleanSess}, _ProtoState)
-    when size(ClientId) == 0 andalso (not CleanSess) ->
+    when byte_size(ClientId) == 0 andalso (not CleanSess) ->
     false;
 
 %% MQTT3.1.1 allow null clientId.
-validate_clientid(#mqtt_packet_connect{proto_ver =?MQTT_PROTO_V311,
+validate_clientid(#mqtt_packet_connect{proto_ver =?MQTT_PROTO_V4,
                                        client_id = ClientId}, _ProtoState)
-    when size(ClientId) =:= 0 ->
+    when byte_size(ClientId) =:= 0 ->
     true;
 
 validate_clientid(#mqtt_packet_connect{proto_ver  = ProtoVer,

src/emqttd_pubsub.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,11 +18,12 @@
 
 -behaviour(gen_server2).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_internal.hrl").
 
-%% Start API.
 -export([start_link/3]).
 
 %% PubSub API.
@@ -31,7 +32,7 @@
 
 -export([dispatch/2]).
 
-%% gen_server.
+%% gen_server Callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
@@ -45,7 +46,6 @@
 %% Start PubSub
 %%--------------------------------------------------------------------
 
-%% @doc Start one Pubsub
 -spec(start_link(atom(), pos_integer(), list()) -> {ok, pid()} | ignore | {error, any()}).
 start_link(Pool, Id, Env) ->
     gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []).
@@ -115,6 +115,7 @@ dispatch({_Share, [Sub]}, Topic, Msg) ->
     dispatch(Sub, Topic, Msg);
 dispatch({_Share, []}, _Topic, _Msg) ->
     ok;
+%%TODO: round-robbin
 dispatch({_Share, Subs}, Topic, Msg) ->
     dispatch(lists:nth(rand:uniform(length(Subs)), Subs), Topic, Msg).
 

src/emqttd_pubsub_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc PubSub Supervisor.
 -module(emqttd_pubsub_sup).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(supervisor).
 
 %% API

src/emqttd_router.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

src/emqttd_serializer.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc MQTT Packet Serializer
 -module(emqttd_serializer).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -25,7 +27,7 @@
 -export([serialize/1]).
 
 %% @doc Serialise MQTT Packet
--spec(serialize(mqtt_packet()) -> binary()).
+-spec(serialize(mqtt_packet()) -> iolist()).
 serialize(#mqtt_packet{header = Header = #mqtt_packet_header{type = Type},
                        variable = Variable,
                        payload  = Payload}) ->
@@ -37,14 +39,12 @@ serialize_header(#mqtt_packet_header{type   = Type,
                                      dup    = Dup,
                                      qos    = Qos,
                                      retain = Retain},
-                 {VariableBin, PayloadBin}) when ?CONNECT =< Type andalso Type =< ?DISCONNECT ->
+                 {VariableBin, PayloadBin})
-    Len = size(VariableBin) + size(PayloadBin),
+    when ?CONNECT =< Type andalso Type =< ?DISCONNECT ->
+    Len = byte_size(VariableBin) + byte_size(PayloadBin),
     true = (Len =< ?MAX_LEN),
-    LenBin = serialize_len(Len),
+    [<<Type:4, (opt(Dup)):1, (opt(Qos)):2, (opt(Retain)):1>>,
-    <<Type:4, (opt(Dup)):1, (opt(Qos)):2, (opt(Retain)):1,
+     serialize_len(Len), VariableBin, PayloadBin].
-      LenBin/binary,
-      VariableBin/binary,
-      PayloadBin/binary>>.
 
 serialize_variable(?CONNECT, #mqtt_packet_connect{client_id   =  ClientId,
                                                   proto_ver   =  ProtoVer,
@@ -58,7 +58,7 @@ serialize_variable(?CONNECT, #mqtt_packet_connect{client_id   =  ClientId,
                                                   will_msg    =  WillMsg,
                                                   username    =  Username,
                                                   password    =  Password}, undefined) ->
-    VariableBin = <<(size(ProtoName)):16/big-unsigned-integer,
+    VariableBin = <<(byte_size(ProtoName)):16/big-unsigned-integer,
                     ProtoName/binary,
                     ProtoVer:8,
                     (opt(Username)):1,
@@ -73,7 +73,7 @@ serialize_variable(?CONNECT, #mqtt_packet_connect{client_id   =  ClientId,
     PayloadBin1 = case WillFlag of
                       true -> <<PayloadBin/binary,
                                 (serialize_utf(WillTopic))/binary,
-                                (size(WillMsg)):16/big-unsigned-integer,
+                                (byte_size(WillMsg)):16/big-unsigned-integer,
                                 WillMsg/binary>>;
                       false -> PayloadBin
                   end,
@@ -93,7 +93,7 @@ serialize_variable(?SUBACK, #mqtt_packet_suback{packet_id = PacketId,
     {<<PacketId:16/big>>, << <<Q:8>> || Q <- QosTable >>};
 
 serialize_variable(?UNSUBSCRIBE, #mqtt_packet_unsubscribe{packet_id  = PacketId,
-                                                          topics = Topics }, undefined) ->
+                                                          topics     = Topics }, undefined) ->
     {<<PacketId:16/big>>, serialize_topics(Topics)};
 
 serialize_variable(?UNSUBACK, #mqtt_packet_unsuback{packet_id = PacketId}, undefined) ->
@@ -134,7 +134,7 @@ serialize_topics([H|_] = Topics) when is_binary(H) ->
 
 serialize_utf(String) ->
     StringBin = unicode:characters_to_binary(String),
-    Len = size(StringBin),
+    Len = byte_size(StringBin),
     true = (Len =< 16#ffff),
     <<Len:16/big, StringBin/binary>>.
 
@@ -148,4 +148,3 @@ opt(false)                -> 0;
 opt(true)                 -> 1;
 opt(X) when is_integer(X) -> X;
 opt(B) when is_binary(B)  -> 1.
-

src/emqttd_server.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@
 
 -module(emqttd_server).
 
--author("Feng Lee <feng@emqtt.io>").
-
 -behaviour(gen_server2).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
@@ -43,7 +43,7 @@
 %% Debug API
 -export([dump/0]).
 
-%% gen_server.
+%% gen_server Callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
@@ -89,7 +89,7 @@ async_subscribe(Topic, Subscriber, Options) when is_binary(Topic) ->
 -spec(publish(mqtt_message()) -> {ok, mqtt_delivery()} | ignore).
 publish(Msg = #mqtt_message{from = From}) ->
     trace(publish, From, Msg),
-    case emqttd_hook:run('message.publish', [], Msg) of
+    case emqttd_hooks:run('message.publish', [], Msg) of
         {ok, Msg1 = #mqtt_message{topic = Topic}} ->
             emqttd_pubsub:publish(Topic, Msg1);
         {stop, Msg1} ->
@@ -97,14 +97,13 @@ publish(Msg = #mqtt_message{from = From}) ->
             ignore
     end.
 
+%% @private
 trace(publish, From, _Msg) when is_atom(From) ->
     %% Dont' trace '$SYS' publish
     ignore;
-
 trace(publish, {ClientId, Username}, #mqtt_message{topic = Topic, payload = Payload}) ->
     lager:info([{client, ClientId}, {topic, Topic}],
                "~s/~s PUBLISH to ~s: ~p", [ClientId, Username, Topic, Payload]);
-
 trace(publish, From, #mqtt_message{topic = Topic, payload = Payload}) when is_binary(From); is_list(From) ->
     lager:info([{client, From}, {topic, Topic}],
                "~s PUBLISH to ~s: ~p", [From, Topic, Payload]).
@@ -142,7 +141,8 @@ subscriptions(Subscriber) ->
 subscription(Topic, Subscriber) ->
     {Topic, Subscriber, ets:lookup_element(mqtt_subproperty, {Topic, Subscriber}, 2)}.
 
-subscribers(Topic) -> emqttd_pubsub:subscribers(Topic).
+subscribers(Topic) ->
+    emqttd_pubsub:subscribers(Topic).
 
 -spec(is_subscribed(binary(), emqttd:subscriber()) -> boolean()).
 is_subscribed(Topic, Subscriber) when is_binary(Topic) ->

src/emqttd_session_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,9 +14,11 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd session supervisor.
+%% @doc Session Supervisor.
 -module(emqttd_session_sup).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behavior(supervisor).
 
 -export([start_link/0, start_session/3]).
@@ -41,4 +43,3 @@ init([]) ->
     {ok, {{simple_one_for_one, 0, 1},
           [{session, {emqttd_session, start_link, []},
               temporary, 5000, worker, [emqttd_session]}]}}.
-

src/emqttd_sm.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,9 +14,10 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc Session Manager
 -module(emqttd_sm).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(gen_server2).
 
 -include("emqttd.hrl").
@@ -32,10 +33,12 @@
 %% API Function Exports
 -export([start_link/2]).
 
--export([start_session/2, lookup_session/1, reg_session/3, unreg_session/1]).
+-export([start_session/2, lookup_session/1, register_session/3, unregister_session/1]).
 
 -export([dispatch/3]).
 
+-export([local_sessions/0]).
+
 %% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
@@ -91,13 +94,13 @@ lookup_session(ClientId) ->
     end.
 
 %% @doc Register a session with info.
--spec(reg_session(binary(), boolean(), [tuple()]) -> true).
+-spec(register_session(binary(), boolean(), [tuple()]) -> true).
-reg_session(ClientId, CleanSess, Properties) ->
+register_session(ClientId, CleanSess, Properties) ->
     ets:insert(mqtt_local_session, {ClientId, self(), CleanSess, Properties}).
 
 %% @doc Unregister a session.
--spec(unreg_session(binary()) -> true).
+-spec(unregister_session(binary()) -> true).
-unreg_session(ClientId) ->
+unregister_session(ClientId) ->
     ets:delete(mqtt_local_session, ClientId).
 
 dispatch(ClientId, Topic, Msg) ->
@@ -110,8 +113,12 @@ dispatch(ClientId, Topic, Msg) ->
 call(SM, Req) ->
     gen_server2:call(SM, Req, ?TIMEOUT). %%infinity).
 
+%% @doc for debug.
+local_sessions() ->
+    ets:tab2list(mqtt_local_session).
+
 %%--------------------------------------------------------------------
-%% gen_server callbacks
+%% gen_server Callbacks
 %%--------------------------------------------------------------------
 
 init([Pool, Id]) ->
@@ -171,6 +178,7 @@ handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) ->
                     [] ->
                         ok;
                     [Sess = #mqtt_session{sess_pid = DownPid}] ->
+                        emqttd_stats:del_session_stats(ClientId),
                         mnesia:delete_object(mqtt_session, Sess, write);
                     [_Sess] ->
                         ok
@@ -208,7 +216,7 @@ create_session({CleanSess, {ClientId, Username}, ClientPid}, State) ->
 create_session(CleanSess, {ClientId, Username}, ClientPid) ->
     case emqttd_session_sup:start_session(CleanSess, {ClientId, Username}, ClientPid) of
         {ok, SessPid} ->
-            Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid, persistent = not CleanSess},
+            Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid, clean_sess = CleanSess},
             case insert_session(Session) of
                 {aborted, {conflict, ConflictPid}} ->
                     %% Conflict with othe node?

src/emqttd_sm_helper.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc Session Helper.
 -module(emqttd_sm_helper).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(gen_server).
 
 -include("emqttd.hrl").

src/emqttd_sm_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,10 +15,13 @@
 %%--------------------------------------------------------------------
 
 %% @doc Session Manager Supervisor.
+
 -module(emqttd_sm_sup).
 
 -behaviour(supervisor).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -define(SM, emqttd_sm).

src/emqttd_stats.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,34 +14,39 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd statistics
 -module(emqttd_stats).
 
--include("emqttd.hrl").
-
 -behaviour(gen_server).
 
--define(SERVER, ?MODULE).
+-author("Feng Lee <feng@emqtt.io>").
+
+-include("emqttd.hrl").
 
 -export([start_link/0, stop/0]).
 
-%% statistics API.
+%% Client and Session Stats
--export([statsfun/1, statsfun/2,
+-export([set_client_stats/2, get_client_stats/1, del_client_stats/1,
-         getstats/0, getstat/1,
+         set_session_stats/2, get_session_stats/1, del_session_stats/1]).
-         setstat/2, setstats/3]).
+
+%% Statistics API.
+-export([statsfun/1, statsfun/2, getstats/0, getstat/1, setstat/2, setstats/3]).
 
 %% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
--record(state, {tick_tref}).
+-record(state, {tick}).
+
+-type(stats() :: list({atom(), non_neg_integer()})).
 
 -define(STATS_TAB, mqtt_stats).
+-define(CLIENT_STATS_TAB, mqtt_client_stats).
+-define(SESSION_STATS_TAB, mqtt_session_stats).
 
 %% $SYS Topics for Clients
 -define(SYSTOP_CLIENTS, [
-    'clients/count',    % clients connected current
+    'clients/count', % clients connected current
-    'clients/max'       % max clients connected
+    'clients/max'    % max clients connected
 ]).
 
 %% $SYS Topics for Sessions
@@ -75,10 +80,40 @@
 %% @doc Start stats server
 -spec(start_link() -> {ok, pid()} | ignore | {error, term()}).
 start_link() ->
-    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
 stop() ->
-    gen_server:call(?SERVER, stop).
+    gen_server:call(?MODULE, stop).
+
+-spec(set_client_stats(binary(), stats()) -> true).
+set_client_stats(ClientId, Stats) ->
+    ets:insert(?CLIENT_STATS_TAB, {ClientId, [{'$ts', emqttd_time:now_secs()}|Stats]}).
+
+-spec(get_client_stats(binary()) -> stats()).
+get_client_stats(ClientId) ->
+    case ets:lookup(?CLIENT_STATS_TAB, ClientId) of
+        [{_, Stats}] -> Stats;
+        [] -> []
+    end.
+
+-spec(del_client_stats(binary()) -> true).
+del_client_stats(ClientId) ->
+    ets:delete(?CLIENT_STATS_TAB, ClientId).
+
+-spec(set_session_stats(binary(), stats()) -> true).
+set_session_stats(ClientId, Stats) ->
+    ets:insert(?SESSION_STATS_TAB, {ClientId, [{'$ts', emqttd_time:now_secs()}|Stats]}).
+
+-spec(get_session_stats(binary()) -> stats()).
+get_session_stats(ClientId) ->
+    case ets:lookup(?SESSION_STATS_TAB, ClientId) of
+        [{_, Stats}] -> Stats;
+        [] -> []
+    end.
+
+-spec(del_session_stats(binary()) -> true).
+del_session_stats(ClientId) ->
+    ets:delete(?SESSION_STATS_TAB, ClientId).
 
 %% @doc Generate stats fun
 -spec(statsfun(Stat :: atom()) -> fun()).
@@ -118,13 +153,14 @@ setstats(Stat, MaxStat, Val) ->
 
 init([]) ->
     emqttd_time:seed(),
-    ets:new(?STATS_TAB, [set, public, named_table, {write_concurrency, true}]),
+    lists:foreach(
+      fun(Tab) ->
+        Tab = ets:new(Tab, [set, public, named_table, {write_concurrency, true}])
+      end, [?STATS_TAB, ?CLIENT_STATS_TAB, ?SESSION_STATS_TAB]),
     Topics = ?SYSTOP_CLIENTS ++ ?SYSTOP_SESSIONS ++ ?SYSTOP_PUBSUB ++ ?SYSTOP_RETAINED,
     ets:insert(?STATS_TAB, [{Topic, 0} || Topic <- Topics]),
-    % Create $SYS Topics
-    % [ok = emqttd:create(topic, stats_topic(Topic)) || Topic <- Topics],
     % Tick to publish stats
-    {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}.
+    {ok, #state{tick = emqttd_broker:start_tick(tick)}, hibernate}.
 
 handle_call(stop, _From, State) ->
     {stop, normal, ok, State};
@@ -154,7 +190,7 @@ handle_info(tick, State) ->
 handle_info(_Info, State) ->
     {noreply, State}.
 
-terminate(_Reason, #state{tick_tref = TRef}) ->
+terminate(_Reason, #state{tick = TRef}) ->
     emqttd_broker:stop_tick(TRef).
 
 code_change(_OldVsn, State, _Extra) ->

src/emqttd_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,11 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc emqttd top supervisor.
 -module(emqttd_sup).
 
 -behaviour(supervisor).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 %% API
@@ -50,5 +51,5 @@ start_child(Mod, Type) when is_atom(Mod) and is_atom(Type) ->
 %%--------------------------------------------------------------------
 
 init([]) ->
-    {ok, {{one_for_all, 10, 3600}, []}}.
+    {ok, {{one_for_all, 0, 1}, []}}.
 

src/emqttd_sysmon.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,6 +17,8 @@
 %% @doc VM System Monitor
 -module(emqttd_sysmon).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behavior(gen_server).
 
 -include("emqttd_internal.hrl").

src/emqttd_sysmon_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

src/emqttd_time.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,21 +16,27 @@
 
 -module(emqttd_time).
 
--export([seed/0, now_to_secs/0, now_to_secs/1, now_to_ms/0, now_to_ms/1]).
+-author("Feng Lee <feng@emqtt.io>").
+
+-export([seed/0, now_secs/0, now_secs/1, now_ms/0, now_ms/1, ts_from_ms/1]).
 
 seed() ->
     case erlang:function_exported(erlang, timestamp, 0) of
-        true  -> random:seed(erlang:timestamp()); %% R18
+        true  -> rand:seed(exsplus, erlang:timestamp()); %% R18
         false -> random:seed(os:timestamp()) %% Compress now() deprecated warning...
     end.
 
-now_to_secs() -> now_to_secs(os:timestamp()).
+now_ms() ->
+    now_ms(os:timestamp()).
 
-now_to_secs({MegaSecs, Secs, _MicroSecs}) ->
+now_ms({MegaSecs, Secs, MicroSecs}) ->
-    MegaSecs * 1000000 + Secs.
-
-now_to_ms() -> now_to_ms(os:timestamp()).
-
-now_to_ms({MegaSecs, Secs, MicroSecs}) ->
     (MegaSecs * 1000000 + Secs) * 1000 + round(MicroSecs/1000).
 
+now_secs() ->
+    now_secs(os:timestamp()).
+
+now_secs({MegaSecs, Secs, _MicroSecs}) ->
+    MegaSecs * 1000000 + Secs.
+
+ts_from_ms(Ms) ->
+    {Ms div 1000000, Ms rem 1000000, 0}.

src/emqttd_topic.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,8 +16,12 @@
 
 -module(emqttd_topic).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd_protocol.hrl").
 
+-include("emqttd_internal.hrl").
+
 -import(lists, [reverse/1]).
 
 -export([match/2, validate/1, triples/1, words/1, wildcard/1]).
@@ -55,8 +59,8 @@ wildcard([_H|T]) ->
 
 %% @doc Match Topic name with filter
 -spec(match(Name, Filter) -> boolean() when
-      Name    :: topic() | words(),
+      Name   :: topic() | words(),
-      Filter  :: topic() | words()).
+      Filter :: topic() | words()).
 match(Name, Filter) when is_binary(Name) and is_binary(Filter) ->
     match(words(Name), words(Filter));
 match([], []) ->
@@ -101,10 +105,7 @@ validate2([''|Words]) ->
 validate2(['+'|Words]) ->
     validate2(Words);
 validate2([W|Words]) ->
-    case validate3(W) of
+    case validate3(W) of true -> validate2(Words); false -> false end.
-        true -> validate2(Words);
-        false -> false
-    end.
 
 validate3(<<>>) ->
     true;
@@ -180,24 +181,28 @@ join(Words) ->
 parse(Topic) when is_binary(Topic) ->
     parse(Topic, []).
 
-parse(Topic = <<"$local/", Topic1/binary>>, Options) ->
+parse(<<"$local/", Topic1/binary>>, Options) ->
-    case lists:member(local, Options) of
+    if_not_contain(local, Options, fun() ->
-        true  -> error({invalid_topic, Topic});
+                       parse(Topic1, [local | Options])
-        false -> parse(Topic1, [local | Options])
+                   end);
-    end;
 
-parse(Topic = <<"$queue/", Topic1/binary>>, Options) ->
+parse(<<"$queue/", Topic1/binary>>, Options) ->
-    case lists:keyfind(share, 1, Options) of
+    if_not_contain(share, Options,fun() ->
-        {share, _} -> error({invalid_topic, Topic});
+                       parse(Topic1, [{share, '$queue'} | Options])
-        false      -> parse(Topic1, [{share, '$queue'} | Options])
+                   end);
-    end;
 
-parse(Topic = <<"$share/", Topic1/binary>>, Options) ->
+parse(<<"$share/", Topic1/binary>>, Options) ->
-    case lists:keyfind(share, 1, Options) of
+    if_not_contain(share, Options, fun() ->
-        {share, _} -> error({invalid_topic, Topic});
+                       [Share, Topic2] = binary:split(Topic1, <<"/">>),
-        false      -> [Share, Topic2] = binary:split(Topic1, <<"/">>),
+                       {Topic2, [{share, Share} | Options]}
-                      {Topic2, [{share, Share} | Options]}
+                   end);
-    end;
 
-parse(Topic, Options) -> {Topic, Options}.
+parse(Topic, Options) ->
+    {Topic, Options}.
+
+if_not_contain(local, Options, Fun) ->
+    ?IF(lists:member(local, Options), error(invalid_topic), Fun());
+
+if_not_contain(share, Options, Fun) ->
+    ?IF(lists:keyfind(share, 1, Options), error(invalid_topic), Fun()).
 

src/emqttd_trace.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -14,13 +14,13 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc
+%% @docTrace MQTT packets/messages by ClientID or Topic.
-%% Trace MQTT packets/messages by ClientID or Topic.
-%% @end
 -module(emqttd_trace).
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd_internal.hrl").
 
 %% API Function Exports
@@ -34,7 +34,7 @@
 
 -record(state, {level, traces}).
 
--type trace_who() :: {client | topic, binary()}.
+-type(trace_who() :: {client | topic, binary()}).
 
 -define(TRACE_OPTIONS, [{formatter_config, [time, " [",severity,"] ", message, "\n"]}]).
 

src/emqttd_trace_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_trace_sup).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(supervisor).
 
 %% API
@@ -29,6 +31,6 @@ start_link() ->
 
 init([]) ->
     Trace = {trace, {emqttd_trace, start_link, []},
-              permanent, 5000, worker, [emqttd_trace]},
+             permanent, 5000, worker, [emqttd_trace]},
-    {ok, {{one_for_one, 10, 100}, [Trace]}}.
+    {ok, {{one_for_one, 10, 3600}, [Trace]}}.
 

src/emqttd_trie.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -17,8 +17,11 @@
 %% @doc MQTT Topic Trie:
 %% [Trie](http://en.wikipedia.org/wiki/Trie)
 %% @end
+
 -module(emqttd_trie).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd_trie.hrl").
 
 %% Mnesia Callbacks

src/emqttd_vm.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

src/emqttd_ws.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(emqttd_ws).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -export([handle_request/1, ws_loop/3]).
 
 %% WebSocket Loop State

src/emqttd_ws_client.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,50 +18,64 @@
 
 -behaviour(gen_server).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -include("emqttd.hrl").
 
 -include("emqttd_protocol.hrl").
 
 %% API Exports
--export([start_link/4, session/1, info/1, kick/1]).
+-export([start_link/4]).
+
+%% Management and Monitor API
+-export([info/1, stats/1, kick/1]).
 
 %% SUB/UNSUB Asynchronously
 -export([subscribe/2, unsubscribe/2]).
 
+%% Get the session proc?
+-export([session/1]).
+
 %% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
 %% WebSocket Client State
--record(wsclient_state, {ws_pid, peer, connection, proto_state, keepalive}).
+-record(wsclient_state, {ws_pid, peer, connection, proto_state, keepalive,
+                         enable_stats, stats_timer}).
+
+-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
 
 -define(WSLOG(Level, Peer, Format, Args),
               lager:Level("WsClient(~s): " ++ Format, [Peer | Args])).
 
 %% @doc Start WebSocket Client.
-start_link(MqttEnv, WsPid, Req, ReplyChannel) ->
+start_link(Env, WsPid, Req, ReplyChannel) ->
-    gen_server:start_link(?MODULE, [MqttEnv, WsPid, Req, ReplyChannel], []).
+    gen_server:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], []).
-
-session(CPid) ->
-    gen_server:call(CPid, session, infinity).
 
 info(CPid) ->
-    gen_server:call(CPid, info, infinity).
+    gen_server:call(CPid, info).
+
+stats(CPid) ->
+    gen_server:call(CPid, stats).
 
 kick(CPid) ->
     gen_server:call(CPid, kick).
 
 subscribe(CPid, TopicTable) ->
-    gen_server:cast(CPid, {subscribe, TopicTable}).
+    CPid ! {subscribe, TopicTable}.
 
 unsubscribe(CPid, Topics) ->
-    gen_server:cast(CPid, {unsubscribe, Topics}).
+    CPid ! {unsubscribe, Topics}.
+
+session(CPid) ->
+    gen_server:call(CPid, session).
 
 %%--------------------------------------------------------------------
 %% gen_server Callbacks
 %%--------------------------------------------------------------------
 
-init([MqttEnv, WsPid, Req, ReplyChannel]) ->
+init([Env, WsPid, Req, ReplyChannel]) ->
     process_flag(trap_exit, true),
     true = link(WsPid),
     {ok, Peername} = Req:get(peername),
@@ -73,45 +87,41 @@ init([MqttEnv, WsPid, Req, ReplyChannel]) ->
                   emqttd_metrics:inc('bytes/sent', iolist_size(Data)),
                   ReplyChannel({binary, Data})
               end,
+    EnableStats = proplists:get_value(client_enable_stats, Env, false),
     ProtoState = emqttd_protocol:init(Peername, SendFun,
-                                      [{ws_initial_headers, Headers} | MqttEnv]),
+                                      [{ws_initial_headers, Headers} | Env]),
-    {ok, #wsclient_state{ws_pid = WsPid, peer = Req:get(peer),
+    {ok, maybe_enable_stats(#wsclient_state{ws_pid       = WsPid,
-                         connection = Req:get(connection),
+                                            peer         = Req:get(peer),
-                         proto_state = ProtoState}, idle_timeout(MqttEnv)}.
+                                            connection   = Req:get(connection),
+                                            proto_state  = ProtoState,
+                                            enable_stats = EnableStats}),
+     proplists:get_value(client_idle_timeout, Env, 30000)}.
 
-idle_timeout(MqttEnv) ->
+handle_call(info, From, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) ->
-    timer:seconds(proplists:get_value(client_idle_timeout, MqttEnv, 10)).
+    Info = [{websocket, true}, {peer, Peer} | emqttd_protocol:info(ProtoState)],
+    {reply, Stats, _} = handle_call(stats, From, State),
+    {reply, lists:append(Info, Stats), State};
 
-handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) ->
+handle_call(stats, _From, State = #wsclient_state{proto_state = ProtoState}) ->
-    {reply, emqttd_protocol:session(ProtoState), State};
+    {reply, lists:append([emqttd_misc:proc_stats(),
-
+                          wsock_stats(State),
-handle_call(info, _From, State = #wsclient_state{peer = Peer,
+                          emqttd_protocol:stats(ProtoState)]), State};
-                                                 proto_state = ProtoState}) ->
-    ProtoInfo = emqttd_protocol:info(ProtoState),
-    {reply, [{websocket, true}, {peer, Peer}| ProtoInfo], State};
 
 handle_call(kick, _From, State) ->
     {stop, {shutdown, kick}, ok, State};
 
+handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) ->
+    {reply, emqttd_protocol:session(ProtoState), State};
+
 handle_call(Req, _From, State = #wsclient_state{peer = Peer}) ->
-    ?WSLOG(critical, Peer, "Unexpected request: ~p", [Req]),
+    ?WSLOG(error, Peer, "Unexpected request: ~p", [Req]),
     {reply, {error, unsupported_request}, State}.
 
-handle_cast({subscribe, TopicTable}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                emqttd_protocol:handle({subscribe, TopicTable}, ProtoState)
-        end, State);
-
-handle_cast({unsubscribe, Topics}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                emqttd_protocol:handle({unsubscribe, Topics}, ProtoState)
-        end, State);
-
 handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) ->
     emqttd_metrics:received(Packet),
     case emqttd_protocol:received(Packet, ProtoState) of
         {ok, ProtoState1} ->
-            noreply(State#wsclient_state{proto_state = ProtoState1});
+            {noreply, State#wsclient_state{proto_state = ProtoState1}, hibernate};
         {error, Error} ->
             ?WSLOG(error, Peer, "Protocol error - ~p", [Error]),
             shutdown(Error, State);
@@ -122,28 +132,46 @@ handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state
     end;
 
 handle_cast(Msg, State = #wsclient_state{peer = Peer}) ->
-    ?WSLOG(critical, Peer, "Unexpected msg: ~p", [Msg]),
+    ?WSLOG(error, Peer, "Unexpected msg: ~p", [Msg]),
-    noreply(State).
+    {noreply, State}.
+
+handle_info({subscribe, TopicTable}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          emqttd_protocol:subscribe(TopicTable, ProtoState)
+      end, State);
+
+handle_info({unsubscribe, Topics}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          emqttd_protocol:unsubscribe(Topics, ProtoState)
+      end, State);
+
+handle_info({suback, PacketId, GrantedQos}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
+          emqttd_protocol:send(Packet, ProtoState)
+      end, State);
+
+handle_info({deliver, Message}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          emqttd_protocol:send(Message, ProtoState)
+      end, State);
+
+handle_info({redeliver, {?PUBREL, PacketId}}, State) ->
+    with_proto(
+      fun(ProtoState) ->
+          emqttd_protocol:pubrel(PacketId, ProtoState)
+      end, State);
+
+handle_info({timeout, _Timer, emit_stats}, State) ->
+    {noreply, maybe_enable_stats(emit_stats(State)), hibernate};
 
 handle_info(timeout, State) ->
     shutdown(idle_timeout, State);
 
-handle_info({suback, PacketId, GrantedQos}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                       Packet = ?SUBACK_PACKET(PacketId, GrantedQos),
-                       emqttd_protocol:send(Packet, ProtoState)
-                     end, State);
-
-handle_info({deliver, Message}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                       emqttd_protocol:send(Message, ProtoState)
-                     end, State);
-
-handle_info({redeliver, {?PUBREL, PacketId}}, State) ->
-    with_proto_state(fun(ProtoState) ->
-                       emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState)
-                     end, State);
-
 handle_info({shutdown, conflict, {ClientId, NewPid}}, State = #wsclient_state{peer = Peer}) ->
     ?WSLOG(warning, Peer, "clientid '~s' conflict with ~p", [ClientId, NewPid]),
     shutdown(conflict, State);
@@ -157,13 +185,13 @@ handle_info({keepalive, start, Interval}, State = #wsclient_state{peer = Peer, c
         end
     end,
     KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}),
-    noreply(State#wsclient_state{keepalive = KeepAlive});
+    {noreply, stats_by_keepalive(State#wsclient_state{keepalive = KeepAlive})};
 
 handle_info({keepalive, check}, State = #wsclient_state{peer      = Peer,
                                                         keepalive = KeepAlive}) ->
     case emqttd_keepalive:check(KeepAlive) of
         {ok, KeepAlive1} ->
-            noreply(State#wsclient_state{keepalive = KeepAlive1});
+            {noreply, emit_stats(State#wsclient_state{keepalive = KeepAlive1}), hibernate};
         {error, timeout} ->
             ?WSLOG(debug, Peer, "Keepalive Timeout!", []),
             shutdown(keepalive_timeout, State);
@@ -180,8 +208,8 @@ handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{peer = Peer, ws_pid
     shutdown(Reason, State);
 
 handle_info(Info, State = #wsclient_state{peer = Peer}) ->
-    ?WSLOG(critical, Peer, "Unexpected Info: ~p", [Info]),
+    ?WSLOG(error, Peer, "Unexpected Info: ~p", [Info]),
-    noreply(State).
+    {noreply, State}.
 
 terminate(Reason, #wsclient_state{proto_state = ProtoState, keepalive = KeepAlive}) ->
     emqttd_keepalive:cancel(KeepAlive),
@@ -199,12 +227,32 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-with_proto_state(Fun, State = #wsclient_state{proto_state = ProtoState}) ->
+maybe_enable_stats(State = #wsclient_state{enable_stats = false}) ->
-    {ok, ProtoState1} = Fun(ProtoState),
+    State;
-    noreply(State#wsclient_state{proto_state = ProtoState1}).
+maybe_enable_stats(State = #wsclient_state{enable_stats = keepalive}) ->
+    State;
+maybe_enable_stats(State = #wsclient_state{enable_stats = Interval}) ->
+    State#wsclient_state{stats_timer = emqttd_misc:start_timer(Interval, self(), emit_stats)}.
 
-noreply(State) ->
+stats_by_keepalive(State) ->
-    {noreply, State, hibernate}.
+    State#wsclient_state{enable_stats = keepalive}.
+
+emit_stats(State = #wsclient_state{enable_stats = false}) ->
+    State;
+emit_stats(State = #wsclient_state{proto_state = ProtoState}) ->
+    {reply, Stats, _} = handle_call(stats, undefined, State),
+    emqttd_stats:set_client_stats(emqttd_protocol:clientid(ProtoState), Stats),
+    State.
+
+wsock_stats(#wsclient_state{connection = Conn}) ->
+    case Conn:getstat(?SOCK_STATS) of
+        {ok,   Ss} -> Ss;
+        {error, _} -> []
+    end.
+
+with_proto(Fun, State = #wsclient_state{proto_state = ProtoState}) ->
+    {ok, ProtoState1} = Fun(ProtoState),
+    {noreply, State#wsclient_state{proto_state = ProtoState1}}.
 
 shutdown(Reason, State) ->
     stop({shutdown, Reason}, State).

src/emqttd_ws_client_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -37,8 +37,9 @@ start_client(WsPid, Req, ReplyChannel) ->
 %%--------------------------------------------------------------------
 %% Supervisor callbacks
 %%--------------------------------------------------------------------
+
 init([]) ->
-    {ok, Env} = emqttd:env(protocol),
+    Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])),
     {ok, {{simple_one_for_one, 0, 1},
            [{ws_client, {emqttd_ws_client, start_link, [Env]},
              temporary, 5000, worker, [emqttd_ws_client]}]}}.

src/lager_emqtt_backend.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2012-2017 Feng Lee <feng@emqtt.io>.
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 -module(lager_emqtt_backend).
 
+-author("Feng Lee <feng@emqtt.io>").
+
 -behaviour(gen_event).
 
 -include_lib("lager/include/lager.hrl").

test/emqttd_inflight_SUITE.erl

@@ -0,0 +1,50 @@
+%%
+%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io)
+%%
+
+-module(emqttd_inflight_SUITE).
+
+-author("Feng Lee <feng@emqtt.io>").
+
+-include_lib("eunit/include/eunit.hrl").
+
+%% CT
+-compile(export_all).
+
+all() -> [t_contain, t_lookup, t_insert, t_update, t_delete, t_window,
+          t_is_full, t_is_empty].
+
+t_contain(_) ->
+    Inflight = emqttd_inflight:new(0),
+    ?assertNot(Inflight:contain(k)),
+    Inflight1 = Inflight:insert(k, v),
+    ?assert(Inflight1:contain(k)).
+
+t_lookup(_) ->
+    Inflight = (emqttd_inflight:new(0)):insert(k, v),
+    ?assertEqual(v, Inflight:lookup(k)).
+
+t_insert(_) ->
+    Inflight = ((emqttd_inflight:new(0)):insert(k1, v1)):insert(k2, v2),
+    ?assertEqual(v2, Inflight:lookup(k2)).
+
+t_update(_) ->
+    Inflight = ((emqttd_inflight:new(0)):insert(k, v1)):update(k, v2),
+    ?assertEqual(v2, Inflight:lookup(k)).
+
+t_delete(_) ->
+    Inflight = ((emqttd_inflight:new(0)):insert(k, v1)):delete(k),
+    ?assert(Inflight:is_empty()).
+
+t_window(_) ->
+    ?assertEqual([], (emqttd_inflight:new(10)):window()),
+    Inflight = ((emqttd_inflight:new(0)):insert(1, 1)):insert(2, 2),
+    ?assertEqual([1, 2], Inflight:window()).
+
+t_is_full(_) ->
+    Inflight = ((emqttd_inflight:new(1)):insert(k, v1)),
+    ?assert(Inflight:is_full()).
+
+t_is_empty(_) ->
+    Inflight = ((emqttd_inflight:new(1)):insert(k, v1)),
+    ?assertNot(Inflight:is_empty()).

test/emqttd_lib_SUITE.erl

@@ -141,8 +141,8 @@ priority_queue_out2(_) ->
 
 time_now_to_(_) ->
     emqttd_time:seed(),
-    emqttd_time:now_to_secs(),
+    emqttd_time:now_secs(),
-    emqttd_time:now_to_ms().
+    emqttd_time:now_ms().
 
 %%--------------------------------------------------------------------
 %% emqttd_node