From cabd1af23aba44c77ffdda4a5b8f79086f5d8816 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Mon, 20 Feb 2017 16:45:53 +0800 Subject: [PATCH 01/41] Hiberate after subscribe, unsubscribe and resume --- src/emqttd_session.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 31efaab55..eb3756460 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -391,7 +391,7 @@ handle_cast({subscribe, _From, TopicTable, AckFun}, {[NewQos|QosAcc], SubMap1} end, {[], Subscriptions}, TopicTable), AckFun(lists:reverse(GrantedQos)), - noreply(emit_stats(State#state{subscriptions = Subscriptions1})); + hibernate(emit_stats(State#state{subscriptions = Subscriptions1})); handle_cast({unsubscribe, _From, TopicTable}, State = #state{client_id = ClientId, @@ -409,7 +409,7 @@ handle_cast({unsubscribe, _From, TopicTable}, SubMap end end, Subscriptions, TopicTable), - noreply(emit_stats(State#state{subscriptions = Subscriptions1})); + hibernate(emit_stats(State#state{subscriptions = Subscriptions1})); %% PUBACK: handle_cast({puback, PacketId}, State = #state{inflight = Inflight}) -> @@ -501,7 +501,7 @@ handle_cast({resume, ClientId, ClientPid}, end, %% Replay delivery and Dequeue pending messages - noreply(emit_stats(dequeue(retry_delivery(true, State1)))); + hibernate(emit_stats(dequeue(retry_delivery(true, State1)))); handle_cast({destroy, ClientId}, State = #state{client_id = ClientId, client_pid = undefined}) -> From 3cdf2377c8d69374770c42870a1a466d5f158a81 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 18:45:40 +0800 Subject: [PATCH 02/41] Use gen_server2 to reduce the CPU/memory usage --- src/emqttd_client.erl | 95 +++++++++++++------------ src/emqttd_misc.erl | 4 +- src/emqttd_protocol.erl | 2 + src/emqttd_session.erl | 150 ++++++++++++++++++--------------------- src/emqttd_ws_client.erl | 86 ++++++++++++---------- 5 files changed, 174 insertions(+), 163 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 10c6f6edc..b48e12f39 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -18,7 +18,7 @@ -module(emqttd_client). --behaviour(gen_server). +-behaviour(gen_server2). -author("Feng Lee "). @@ -48,11 +48,13 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]). +%% gen_server2 Callbacks +-export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). + %% Client State -record(client_state, {connection, connname, peername, peerhost, peerport, await_recv, conn_state, rate_limit, parser_fun, - proto_state, packet_opts, keepalive, enable_stats, - stats_timer}). + proto_state, packet_opts, keepalive, enable_stats}). -define(INFO_KEYS, [connname, peername, peerhost, peerport, await_recv, conn_state]). @@ -65,19 +67,19 @@ start_link(Conn, Env) -> {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]])}. info(CPid) -> - gen_server:call(CPid, info). + gen_server2:call(CPid, info). stats(CPid) -> - gen_server:call(CPid, stats). + gen_server2:call(CPid, stats). kick(CPid) -> - gen_server:call(CPid, kick). + gen_server2:call(CPid, kick). set_rate_limit(Cpid, Rl) -> - gen_server:call(Cpid, {set_rate_limit, Rl}). + gen_server2:call(Cpid, {set_rate_limit, Rl}). get_rate_limit(Cpid) -> - gen_server:call(Cpid, get_rate_limit). + gen_server2:call(Cpid, get_rate_limit). subscribe(CPid, TopicTable) -> CPid ! {subscribe, TopicTable}. @@ -135,30 +137,41 @@ init([Conn0, Env]) -> packet_opts = Env, enable_stats = EnableStats}), IdleTimout = get_value(client_idle_timeout, Env, 30000), - gen_server:enter_loop(?MODULE, [], maybe_enable_stats(State), IdleTimout). + gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, + {backoff, 1000, 1000, 5000}). + +prioritise_call(Msg, _From, _Len, _State) -> + case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. + +prioritise_info(Msg, _Len, _State) -> + case Msg of {redeliver, _} -> 5; _ -> 0 end. + +handle_pre_hibernate(State = #client_state{connname = Connname}) -> + io:format("Client(~s) will hibernate!~n", [Connname]), + {hibernate, emit_stats(State)}. handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> ProtoInfo = emqttd_protocol:info(ProtoState), ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS), - {reply, Stats, _} = handle_call(stats, From, State), - {reply, lists:append([ClientInfo, ProtoInfo, Stats]), State}; + {reply, Stats, _, _} = handle_call(stats, From, State), + reply(lists:append([ClientInfo, ProtoInfo, Stats]), State); handle_call(stats, _From, State = #client_state{proto_state = ProtoState}) -> - {reply, lists:append([emqttd_misc:proc_stats(), - emqttd_protocol:stats(ProtoState), - sock_stats(State)]), State}; + reply(lists:append([emqttd_misc:proc_stats(), + emqttd_protocol:stats(ProtoState), + sock_stats(State)]), State); handle_call(kick, _From, State) -> {stop, {shutdown, kick}, ok, State}; handle_call({set_rate_limit, Rl}, _From, State) -> - {reply, ok, State#client_state{rate_limit = Rl}}; + reply(ok, State#client_state{rate_limit = Rl}); handle_call(get_rate_limit, _From, State = #client_state{rate_limit = Rl}) -> - {reply, Rl, State}; + reply(Rl, State); handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> - {reply, emqttd_protocol:session(ProtoState), State}; + reply(emqttd_protocol:session(ProtoState), State); handle_call(Req, _From, State) -> ?UNEXPECTED_REQ(Req, State). @@ -198,12 +211,12 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State) -> emqttd_protocol:pubrel(PacketId, ProtoState) end, State); +handle_info(emit_stats, State) -> + {noreply, emit_stats(State), hibernate}; + handle_info(timeout, State) -> shutdown(idle_timeout, State); -handle_info({timeout, _Timer, emit_stats}, State) -> - hibernate(maybe_enable_stats(emit_stats(State))); - %% Fix issue #535 handle_info({shutdown, Error}, State) -> shutdown(Error, State); @@ -213,7 +226,7 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> shutdown(conflict, State); handle_info(activate_sock, State) -> - hibernate(run_socket(State#client_state{conn_state = running})); + {noreply, run_socket(State#client_state{conn_state = running}), hibernate}; handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> Size = iolist_size(Data), @@ -239,12 +252,12 @@ handle_info({keepalive, start, Interval}, State = #client_state{connection = Con end end, KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}), - {noreply, stats_by_keepalive(State#client_state{keepalive = KeepAlive})}; + {noreply, State#client_state{keepalive = KeepAlive}, hibernate}; handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> - hibernate(emit_stats(State#client_state{keepalive = KeepAlive1})); + {noreply, State#client_state{keepalive = KeepAlive1}, hibernate}; {error, timeout} -> ?LOG(debug, "Keepalive timeout", [], State), shutdown(keepalive_timeout, State); @@ -279,7 +292,7 @@ code_change(_OldVsn, State, _Extra) -> %% Receive and parse tcp data received(<<>>, State) -> - hibernate(State); + {noreply, State, hibernate}; received(Bytes, State = #client_state{parser_fun = ParserFun, packet_opts = PacketOpts, @@ -332,33 +345,25 @@ run_socket(State = #client_state{connection = Conn}) -> with_proto(Fun, State = #client_state{proto_state = ProtoState}) -> {ok, ProtoState1} = Fun(ProtoState), - {noreply, State#client_state{proto_state = ProtoState1}}. + {noreply, State#client_state{proto_state = ProtoState1}, hibernate}. -maybe_enable_stats(State = #client_state{enable_stats = false}) -> - State; -maybe_enable_stats(State = #client_state{enable_stats = keepalive}) -> - State; -maybe_enable_stats(State = #client_state{enable_stats = Interval}) -> - State#client_state{stats_timer = emqttd_misc:start_timer(Interval, self(), emit_stats)}. - -stats_by_keepalive(State) -> - State#client_state{enable_stats = keepalive}. - -emit_stats(State = #client_state{enable_stats = false}) -> - State; emit_stats(State = #client_state{proto_state = ProtoState}) -> - {reply, Stats, _} = handle_call(stats, undefined, State), - emqttd_stats:set_client_stats(emqttd_protocol:clientid(ProtoState), Stats), + emit_stats(emqttd_protocol:clientid(ProtoState), State). + +emit_stats(_ClientId, State = #client_state{enable_stats = false}) -> + State; +emit_stats(undefined, State) -> + State; +emit_stats(ClientId, State) -> + {reply, Stats, _, _} = handle_call(stats, undefined, State), + emqttd_stats:set_client_stats(ClientId, Stats), State. sock_stats(#client_state{connection = Conn}) -> - case Conn:getstat(?SOCK_STATS) of - {ok, Ss} -> Ss; - {error, _} -> [] - end. + case Conn:getstat(?SOCK_STATS) of {ok, Ss} -> Ss; {error, _} -> [] end. -hibernate(State) -> - {noreply, State, hibernate}. +reply(Reply, State) -> + {reply, Reply, State, hibernate}. shutdown(Reason, State) -> stop({shutdown, Reason}, State). diff --git a/src/emqttd_misc.erl b/src/emqttd_misc.erl index 8111264cc..69cba9d8f 100644 --- a/src/emqttd_misc.erl +++ b/src/emqttd_misc.erl @@ -19,7 +19,7 @@ -author("Feng Lee "). -export([merge_opts/2, start_timer/2, start_timer/3, cancel_timer/1, - proc_stats/0, proc_stats/1]). + proc_stats/0, proc_stats/1, inc_stats/1]). %% @doc Merge Options merge_opts(Defaults, Options) -> @@ -61,3 +61,5 @@ proc_stats(Pid) -> {value, {_, V}, Stats1} = lists:keytake(message_queue_len, 1, Stats), [{mailbox_len, V} | Stats1]. +inc_stats(Key) -> put(Key, get(Key) + 1). + diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index a9cc37778..da235a44e 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -187,6 +187,8 @@ process(Packet = ?CONNECT_PACKET(Var), State0) -> emqttd_cm:reg(client(State2)), %% Start keepalive start_keepalive(KeepAlive), + %% Emit Stats + self() ! emit_stats, %% ACCEPT {?CONNACK_ACCEPT, SP, State2#proto_state{session = Session, is_superuser = IsSuperuser}}; {error, Error} -> diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index eb3756460..26d7c7a83 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -74,7 +74,8 @@ terminate/2, code_change/3]). %% gen_server2 Message Priorities --export([prioritise_call/4, prioritise_cast/3, prioritise_info/3]). +-export([prioritise_call/4, prioritise_cast/3, prioritise_info/3, + handle_pre_hibernate/1]). -record(state, { @@ -114,7 +115,7 @@ max_inflight = 32 :: non_neg_integer(), %% Retry interval for redelivering QoS1/2 messages - retry_interval = 20000 :: pos_integer(), + retry_interval = 20000 :: timeout(), %% Retry Timer retry_timer :: reference(), @@ -129,7 +130,7 @@ awaiting_rel :: map(), %% Awaiting PUBREL timeout - await_rel_timeout = 20000 :: pos_integer(), + await_rel_timeout = 20000 :: timeout(), %% Max Packets that Awaiting PUBREL max_awaiting_rel = 100 :: non_neg_integer(), @@ -138,16 +139,13 @@ await_rel_timer :: reference(), %% Session Expiry Interval - expiry_interval = 7200000 :: pos_integer(), + expiry_interval = 7200000 :: timeout(), %% Expired Timer expiry_timer :: reference(), %% Enable Stats - enable_stats :: false | pos_integer(), - - %% Stats Timer - stats_timer :: reference(), + enable_stats :: boolean(), created_at :: erlang:timestamp() }). @@ -301,10 +299,9 @@ init([CleanSess, {ClientId, Username}, ClientPid]) -> expiry_interval = get_value(expiry_interval, Env), enable_stats = EnableStats, created_at = os:timestamp()}, - emqttd_stats:set_session_stats(ClientId, stats(State)), emqttd_sm:register_session(ClientId, CleanSess, info(State)), emqttd_hooks:run('session.created', [ClientId, Username]), - {ok, State, hibernate, {backoff, 1000, 1000, 5000}, ?MODULE}. + {ok, emit_stats(State), hibernate, {backoff, 1000, 1000, 10000}, ?MODULE}. init_stats(Keys) -> lists:foreach(fun(K) -> put(K, 0) end, Keys). @@ -336,10 +333,13 @@ prioritise_info(Msg, _Len, _State) -> _ -> 0 end. -handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, - _From, State = #state{awaiting_rel = AwaitingRel, - await_rel_timer = Timer, - await_rel_timeout = Timeout}) -> +handle_pre_hibernate(State) -> + {hibernate, emit_stats(State)}. + +handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, + State = #state{awaiting_rel = AwaitingRel, + await_rel_timer = Timer, + await_rel_timeout = Timeout}) -> case is_awaiting_full(State) of false -> State1 = case Timer == undefined of @@ -413,51 +413,55 @@ handle_cast({unsubscribe, _From, TopicTable}, %% PUBACK: handle_cast({puback, PacketId}, State = #state{inflight = Inflight}) -> - case Inflight:contain(PacketId) of - true -> - noreply(dequeue(acked(puback, PacketId, State))); - false -> - ?LOG(warning, "The PUBACK ~p is not inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/puback/missed'), - noreply(State) - end; + {noreply, + case Inflight:contain(PacketId) of + true -> + dequeue(acked(puback, PacketId, State)); + false -> + ?LOG(warning, "PUBACK ~p missed inflight: ~p", + [PacketId, Inflight:window()], State), + emqttd_metrics:inc('packets/puback/missed'), + State + end, hibernate}; %% PUBREC: handle_cast({pubrec, PacketId}, State = #state{inflight = Inflight}) -> - case Inflight:contain(PacketId) of - true -> - noreply(acked(pubrec, PacketId, State)); - false -> - ?LOG(warning, "The PUBREC ~p is not inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/pubrec/missed'), - noreply(State) - end; + {noreply, + case Inflight:contain(PacketId) of + true -> + acked(pubrec, PacketId, State); + false -> + ?LOG(warning, "PUBREC ~p missed inflight: ~p", + [PacketId, Inflight:window()], State), + emqttd_metrics:inc('packets/pubrec/missed'), + State + end, hibernate}; %% PUBREL: handle_cast({pubrel, PacketId}, State = #state{awaiting_rel = AwaitingRel}) -> - case maps:take(PacketId, AwaitingRel) of - {Msg, AwaitingRel1} -> - spawn(emqttd_server, publish, [Msg]),%%:) - noreply(State#state{awaiting_rel = AwaitingRel1}); - error -> - ?LOG(warning, "Cannot find PUBREL: ~p", [PacketId], State), - emqttd_metrics:inc('packets/pubrel/missed'), - noreply(State) - end; + {noreply, + case maps:take(PacketId, AwaitingRel) of + {Msg, AwaitingRel1} -> + spawn(emqttd_server, publish, [Msg]), %%:) + State#state{awaiting_rel = AwaitingRel1}; + error -> + ?LOG(warning, "Cannot find PUBREL: ~p", [PacketId], State), + emqttd_metrics:inc('packets/pubrel/missed'), + State + end, hibernate}; %% PUBCOMP: handle_cast({pubcomp, PacketId}, State = #state{inflight = Inflight}) -> - case Inflight:contain(PacketId) of - true -> - noreply(dequeue(acked(pubcomp, PacketId, State))); - false -> - ?LOG(warning, "The PUBCOMP ~p is not inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/pubcomp/missed'), - noreply(State) - end; + {noreply, + case Inflight:contain(PacketId) of + true -> + dequeue(acked(pubcomp, PacketId, State)); + false -> + ?LOG(warning, "The PUBCOMP ~p is not inflight: ~p", + [PacketId, Inflight:window()], State), + emqttd_metrics:inc('packets/pubcomp/missed'), + State + end, hibernate}; %% RESUME: handle_cast({resume, ClientId, ClientPid}, @@ -466,14 +470,13 @@ handle_cast({resume, ClientId, ClientPid}, clean_sess = CleanSess, retry_timer = RetryTimer, await_rel_timer = AwaitTimer, - stats_timer = StatsTimer, expiry_timer = ExpireTimer}) -> ?LOG(info, "Resumed by ~p", [ClientPid], State), %% Cancel Timers lists:foreach(fun emqttd_misc:cancel_timer/1, - [RetryTimer, AwaitTimer, StatsTimer, ExpireTimer]), + [RetryTimer, AwaitTimer, ExpireTimer]), case kick(ClientId, OldClientPid, ClientPid) of ok -> ?LOG(warning, "~p kickout ~p", [ClientPid, OldClientPid], State); @@ -503,13 +506,13 @@ handle_cast({resume, ClientId, ClientPid}, %% Replay delivery and Dequeue pending messages hibernate(emit_stats(dequeue(retry_delivery(true, State1)))); -handle_cast({destroy, ClientId}, State = #state{client_id = ClientId, - client_pid = undefined}) -> +handle_cast({destroy, ClientId}, + State = #state{client_id = ClientId, client_pid = undefined}) -> ?LOG(warning, "Destroyed", [], State), shutdown(destroy, State); -handle_cast({destroy, ClientId}, State = #state{client_id = ClientId, - client_pid = OldClientPid}) -> +handle_cast({destroy, ClientId}, + State = #state{client_id = ClientId, client_pid = OldClientPid}) -> ?LOG(warning, "kickout ~p", [OldClientPid], State), shutdown(conflict, State); @@ -518,27 +521,25 @@ handle_cast(Msg, State) -> %% Dispatch Message handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, mqtt_message) -> - noreply(dispatch(tune_qos(Topic, Msg, State), State)); + {noreply, dispatch(tune_qos(Topic, Msg, State), State), hibernate}; %% Do nothing if the client has been disconnected. handle_info({timeout, _Timer, retry_delivery}, State = #state{client_pid = undefined}) -> hibernate(emit_stats(State#state{retry_timer = undefined})); handle_info({timeout, _Timer, retry_delivery}, State) -> - noreply(emit_stats(retry_delivery(false, State#state{retry_timer = undefined}))); + hibernate(emit_stats(retry_delivery(false, State#state{retry_timer = undefined}))); handle_info({timeout, _Timer, check_awaiting_rel}, State) -> - noreply(expire_awaiting_rel(emit_stats(State#state{await_rel_timer = undefined}))); - -handle_info({timeout, _Timer, emit_stats}, State) -> - hibernate(maybe_enable_stats(emit_stats(State))); + hibernate(expire_awaiting_rel(emit_stats(State#state{await_rel_timer = undefined}))); handle_info({timeout, _Timer, expired}, State) -> ?LOG(info, "Expired, shutdown now.", [], State), shutdown(expired, State); handle_info({'EXIT', ClientPid, _Reason}, - State = #state{clean_sess = true, client_pid = ClientPid}) -> + State = #state{clean_sess = true, + client_pid = ClientPid}) -> {stop, normal, State}; handle_info({'EXIT', ClientPid, Reason}, @@ -548,7 +549,7 @@ handle_info({'EXIT', ClientPid, Reason}, ?LOG(info, "Client ~p EXIT for ~p", [ClientPid, Reason], State), ExpireTimer = start_timer(Interval, expired), State1 = State#state{client_pid = undefined, expiry_timer = ExpireTimer}, - hibernate(maybe_enable_stats(emit_stats(State1))); + hibernate(emit_stats(State1)); handle_info({'EXIT', Pid, _Reason}, State = #state{old_client_pid = Pid}) -> %%ignore @@ -690,7 +691,8 @@ dispatch(Msg = #mqtt_message{qos = QoS}, end. enqueue_msg(Msg, State = #state{mqueue = Q}) -> - inc(enqueue_msg), State#state{mqueue = emqttd_mqueue:in(Msg, Q)}. + emqttd_misc:inc_stats(enqueue_msg), + State#state{mqueue = emqttd_mqueue:in(Msg, Q)}. %%-------------------------------------------------------------------- %% Deliver @@ -700,7 +702,8 @@ redeliver(Msg = #mqtt_message{qos = QoS}, State) -> deliver(Msg#mqtt_message{dup = if QoS =:= ?QOS2 -> false; true -> true end}, State). deliver(Msg, #state{client_pid = Pid}) -> - inc(deliver_msg), Pid ! {deliver, Msg}. + emqttd_misc:inc_stats(deliver_msg), + Pid ! {deliver, Msg}. %%-------------------------------------------------------------------- %% Awaiting ACK for QoS1/QoS2 Messages @@ -785,14 +788,6 @@ next_msg_id(State = #state{next_msg_id = Id}) -> %% Emit session stats %%-------------------------------------------------------------------- -maybe_enable_stats(State = #state{enable_stats = false}) -> - State; -maybe_enable_stats(State = #state{client_pid = Pid}) when is_pid(Pid) -> - State; -maybe_enable_stats(State = #state{enable_stats = Interval}) -> - StatsTimer = start_timer(Interval, emit_stats), - State#state{stats_timer = StatsTimer}. - emit_stats(State = #state{enable_stats = false}) -> State; emit_stats(State = #state{client_id = ClientId}) -> @@ -803,13 +798,8 @@ emit_stats(State = #state{client_id = ClientId}) -> %% Helper functions %%-------------------------------------------------------------------- -inc(Key) -> put(Key, get(Key) + 1). - reply(Reply, State) -> - {reply, Reply, State}. - -noreply(State) -> - {noreply, State}. + {reply, Reply, State, hibernate}. hibernate(State) -> {noreply, State, hibernate}. diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 3641a98a8..e2d605bf2 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -16,7 +16,7 @@ -module(emqttd_ws_client). --behaviour(gen_server). +-behaviour(gen_server2). -author("Feng Lee "). @@ -40,9 +40,12 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +%% gen_server2 Callbacks +-export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). + %% WebSocket Client State -record(wsclient_state, {ws_pid, peer, connection, proto_state, keepalive, - enable_stats, stats_timer}). + enable_stats}). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -54,13 +57,13 @@ start_link(Env, WsPid, Req, ReplyChannel) -> gen_server:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], []). info(CPid) -> - gen_server:call(CPid, info). + gen_server2:call(CPid, info). stats(CPid) -> - gen_server:call(CPid, stats). + gen_server2:call(CPid, stats). kick(CPid) -> - gen_server:call(CPid, kick). + gen_server2:call(CPid, kick). subscribe(CPid, TopicTable) -> CPid ! {subscribe, TopicTable}. @@ -69,7 +72,7 @@ unsubscribe(CPid, Topics) -> CPid ! {unsubscribe, Topics}. session(CPid) -> - gen_server:call(CPid, session). + gen_server2:call(CPid, session). %%-------------------------------------------------------------------- %% gen_server Callbacks @@ -90,28 +93,39 @@ init([Env, WsPid, Req, ReplyChannel]) -> EnableStats = proplists:get_value(client_enable_stats, Env, false), ProtoState = emqttd_protocol:init(Peername, SendFun, [{ws_initial_headers, Headers} | Env]), - {ok, maybe_enable_stats(#wsclient_state{ws_pid = WsPid, - peer = Req:get(peer), - connection = Req:get(connection), - proto_state = ProtoState, - enable_stats = EnableStats}), - proplists:get_value(client_idle_timeout, Env, 30000)}. + IdleTimeout = proplists:get_value(client_idle_timeout, Env, 30000), + {ok, #wsclient_state{ws_pid = WsPid, + peer = Req:get(peer), + connection = Req:get(connection), + proto_state = ProtoState, + enable_stats = EnableStats}, + IdleTimeout, {backoff, 1000, 1000, 5000}, ?MODULE}. + +prioritise_call(Msg, _From, _Len, _State) -> + case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. + +prioritise_info(Msg, _Len, _State) -> + case Msg of {redeliver, _} -> 5; _ -> 0 end. + +handle_pre_hibernate(State = #wsclient_state{peer = Peer}) -> + io:format("WsClient(~s) will hibernate!~n", [Peer]), + {hibernate, emit_stats(State)}. handle_call(info, From, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) -> Info = [{websocket, true}, {peer, Peer} | emqttd_protocol:info(ProtoState)], - {reply, Stats, _} = handle_call(stats, From, State), - {reply, lists:append(Info, Stats), State}; + {reply, Stats, _, _} = handle_call(stats, From, State), + reply(lists:append(Info, Stats), State); handle_call(stats, _From, State = #wsclient_state{proto_state = ProtoState}) -> - {reply, lists:append([emqttd_misc:proc_stats(), - wsock_stats(State), - emqttd_protocol:stats(ProtoState)]), State}; + reply(lists:append([emqttd_misc:proc_stats(), + wsock_stats(State), + emqttd_protocol:stats(ProtoState)]), State); handle_call(kick, _From, State) -> {stop, {shutdown, kick}, ok, State}; handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) -> - {reply, emqttd_protocol:session(ProtoState), State}; + reply(emqttd_protocol:session(ProtoState), State); handle_call(Req, _From, State = #wsclient_state{peer = Peer}) -> ?WSLOG(error, Peer, "Unexpected request: ~p", [Req]), @@ -166,8 +180,8 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State) -> emqttd_protocol:pubrel(PacketId, ProtoState) end, State); -handle_info({timeout, _Timer, emit_stats}, State) -> - {noreply, maybe_enable_stats(emit_stats(State)), hibernate}; +handle_info(emit_stats, State) -> + {noreply, emit_stats(State), hibernate}; handle_info(timeout, State) -> shutdown(idle_timeout, State); @@ -185,7 +199,7 @@ handle_info({keepalive, start, Interval}, State = #wsclient_state{peer = Peer, c end end, KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}), - {noreply, stats_by_keepalive(State#wsclient_state{keepalive = KeepAlive})}; + {noreply, State#wsclient_state{keepalive = KeepAlive}, hibernate}; handle_info({keepalive, check}, State = #wsclient_state{peer = Peer, keepalive = KeepAlive}) -> @@ -209,7 +223,7 @@ handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{peer = Peer, ws_pid handle_info(Info, State = #wsclient_state{peer = Peer}) -> ?WSLOG(error, Peer, "Unexpected Info: ~p", [Info]), - {noreply, State}. + {noreply, State, hibernate}. terminate(Reason, #wsclient_state{proto_state = ProtoState, keepalive = KeepAlive}) -> emqttd_keepalive:cancel(KeepAlive), @@ -227,21 +241,16 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -maybe_enable_stats(State = #wsclient_state{enable_stats = false}) -> - State; -maybe_enable_stats(State = #wsclient_state{enable_stats = keepalive}) -> - State; -maybe_enable_stats(State = #wsclient_state{enable_stats = Interval}) -> - State#wsclient_state{stats_timer = emqttd_misc:start_timer(Interval, self(), emit_stats)}. - -stats_by_keepalive(State) -> - State#wsclient_state{enable_stats = keepalive}. - -emit_stats(State = #wsclient_state{enable_stats = false}) -> - State; emit_stats(State = #wsclient_state{proto_state = ProtoState}) -> - {reply, Stats, _} = handle_call(stats, undefined, State), - emqttd_stats:set_client_stats(emqttd_protocol:clientid(ProtoState), Stats), + emit_stats(emqttd_protocol:clientid(ProtoState), State). + +emit_stats(_ClientId, State = #wsclient_state{enable_stats = false}) -> + State; +emit_stats(undefined, State) -> + State; +emit_stats(ClientId, State) -> + {reply, Stats, _, _} = handle_call(stats, undefined, State), + emqttd_stats:set_client_stats(ClientId, Stats), State. wsock_stats(#wsclient_state{connection = Conn}) -> @@ -252,7 +261,10 @@ wsock_stats(#wsclient_state{connection = Conn}) -> with_proto(Fun, State = #wsclient_state{proto_state = ProtoState}) -> {ok, ProtoState1} = Fun(ProtoState), - {noreply, State#wsclient_state{proto_state = ProtoState1}}. + {noreply, State#wsclient_state{proto_state = ProtoState1}, hibernate}. + +reply(Reply, State) -> + {reply, Reply, State, hibernate}. shutdown(Reason, State) -> stop({shutdown, Reason}, State). From ff60578a7da03281b95c95c208abca870171cb4b Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 18:46:10 +0800 Subject: [PATCH 03/41] Change the datatype of 'enable_stats' to 'flag' --- etc/emq.conf | 8 ++++---- priv/emq.schema | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/etc/emq.conf b/etc/emq.conf index e30ddfa26..0e20dd60d 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -113,8 +113,8 @@ mqtt.max_packet_size = 64KB ## Client Idle Timeout (Second) mqtt.client.idle_timeout = 30s -## Enable client Stats: seconds or off -mqtt.client.enable_stats = off +## Enable client Stats: on | off +mqtt.client.enable_stats = on ##-------------------------------------------------------------------- ## MQTT Session @@ -136,8 +136,8 @@ mqtt.session.max_awaiting_rel = 100 ## Awaiting PUBREL Timeout mqtt.session.await_rel_timeout = 20s -## Enable Statistics at the Interval(seconds) -mqtt.session.enable_stats = off +## Enable Statistics: on | off +mqtt.session.enable_stats = on ## Expired after 1 day: ## w - week diff --git a/priv/emq.schema b/priv/emq.schema index 156bc4874..1e3ba1778 100644 --- a/priv/emq.schema +++ b/priv/emq.schema @@ -329,7 +329,7 @@ end}. %% @doc Enable Stats of Client. {mapping, "mqtt.client.enable_stats", "emqttd.client", [ {default, off}, - {datatype, [{duration, ms}, flag]} + {datatype, flag} ]}. %% @doc Client @@ -375,7 +375,7 @@ end}. %% @doc Enable Stats {mapping, "mqtt.session.enable_stats", "emqttd.session", [ {default, off}, - {datatype, [{duration, ms}, flag]} + {datatype, flag} ]}. %% @doc Session Expiry Interval From d310fff532e41f48eb1ab3c9fbc9cf16b8033aae Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 18:46:44 +0800 Subject: [PATCH 04/41] Add 'lager_syslog' dependency --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 3ac69aea2..0b43df393 100644 --- a/rebar.config +++ b/rebar.config @@ -1,4 +1,4 @@ {deps, [ -{gproc,".*",{git,"https://github.com/uwiger/gproc",""}},{lager,".*",{git,"https://github.com/basho/lager","master"}},{esockd,".*",{git,"https://github.com/emqtt/esockd","master"}},{mochiweb,".*",{git,"https://github.com/emqtt/mochiweb",""}} +{gproc,".*",{git,"https://github.com/uwiger/gproc",""}},{lager,".*",{git,"https://github.com/basho/lager","master"}},{esockd,".*",{git,"https://github.com/emqtt/esockd","master"}},{mochiweb,".*",{git,"https://github.com/emqtt/mochiweb",""}},{lager_syslog,".*",{git,"https://github.com/basho/lager_syslog",""}} ]}. {erl_opts, [{parse_transform,lager_transform}]}. From 498915e5b3b663a529e742b5ab3612727d656b9d Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 19:26:50 +0800 Subject: [PATCH 05/41] Hibernate after a packet sent --- src/emqttd_client.erl | 4 ++-- src/emqttd_session.erl | 3 ++- src/emqttd_ws_client.erl | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index b48e12f39..765c9d837 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -238,7 +238,7 @@ handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> shutdown(Reason, State); handle_info({inet_reply, _Sock, ok}, State) -> - {noreply, State}; + {noreply, State, hibernate}; handle_info({inet_reply, _Sock, {error, Reason}}, State) -> shutdown(Reason, State); @@ -299,7 +299,7 @@ received(Bytes, State = #client_state{parser_fun = ParserFun, proto_state = ProtoState}) -> case catch ParserFun(Bytes) of {more, NewParser} -> - {noreply, run_socket(State#client_state{parser_fun = NewParser})}; + {noreply, run_socket(State#client_state{parser_fun = NewParser}), hibernate}; {ok, Packet, Rest} -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 26d7c7a83..852f46c5f 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -333,7 +333,8 @@ prioritise_info(Msg, _Len, _State) -> _ -> 0 end. -handle_pre_hibernate(State) -> +handle_pre_hibernate(State = #state{client_id = ClientId}) -> + io:format("Session(~s) will hibernate!~n", [ClientId]), {hibernate, emit_stats(State)}. handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index e2d605bf2..71cb4b344 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -129,7 +129,7 @@ handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) - handle_call(Req, _From, State = #wsclient_state{peer = Peer}) -> ?WSLOG(error, Peer, "Unexpected request: ~p", [Req]), - {reply, {error, unsupported_request}, State}. + reply({error, unsupported_request}, State). handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) -> emqttd_metrics:received(Packet), @@ -147,7 +147,7 @@ handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state handle_cast(Msg, State = #wsclient_state{peer = Peer}) -> ?WSLOG(error, Peer, "Unexpected msg: ~p", [Msg]), - {noreply, State}. + {noreply, State, hibernate}. handle_info({subscribe, TopicTable}, State) -> with_proto( From de12c58af0872803726e85d03da215c20b402d92 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 20:09:31 +0800 Subject: [PATCH 06/41] Rename 'MAX_LEN' to 'MAX_PACKET_LEN' --- include/emqttd_protocol.hrl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/emqttd_protocol.hrl b/include/emqttd_protocol.hrl index ab0650ead..a7eb768e3 100644 --- a/include/emqttd_protocol.hrl +++ b/include/emqttd_protocol.hrl @@ -129,11 +129,16 @@ -type(mqtt_connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH). +%%-------------------------------------------------------------------- +%% Max MQTT Packet Length +%%-------------------------------------------------------------------- + +-define(MAX_PACKET_LEN, 16#fffffff). + %%-------------------------------------------------------------------- %% MQTT Parser and Serializer %%-------------------------------------------------------------------- --define(MAX_LEN, 16#fffffff). -define(HIGHBIT, 2#10000000). -define(LOWBITS, 2#01111111). From 7e9865023375b3546873c144b4f597fb69b2b8c9 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 20:10:17 +0800 Subject: [PATCH 07/41] Improve the emqttd_parser design --- src/emqttd_client.erl | 27 ++++++++++++++------------- src/emqttd_parser.erl | 29 ++++++++++++----------------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 765c9d837..e58f55738 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -52,9 +52,9 @@ -export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). %% Client State --record(client_state, {connection, connname, peername, peerhost, peerport, - await_recv, conn_state, rate_limit, parser_fun, - proto_state, packet_opts, keepalive, enable_stats}). +-record(client_state, {connection, connname, peername, peerhost, peerport, await_recv, + conn_state, rate_limit, packet_limit, parse_state, proto_state, + keepalive, enable_stats}). -define(INFO_KEYS, [connname, peername, peerhost, peerport, await_recv, conn_state]). @@ -120,9 +120,10 @@ init([Conn0, Env]) -> error:Error -> Self ! {shutdown, Error} end end, - ParserFun = emqttd_parser:new(Env), - ProtoState = emqttd_protocol:init(PeerName, SendFun, Env), RateLimit = get_value(rate_limit, Conn:opts()), + PacketLimit = proplists:get_value(max_packet_size, Env, ?MAX_PACKET_LEN), + ParseState = emqttd_parser:initial_state(PacketLimit), + ProtoState = emqttd_protocol:init(PeerName, SendFun, Env), EnableStats = get_value(client_enable_stats, Env, false), State = run_socket(#client_state{connection = Conn, connname = ConnName, @@ -132,9 +133,9 @@ init([Conn0, Env]) -> await_recv = false, conn_state = running, rate_limit = RateLimit, - parser_fun = ParserFun, + packet_limit = PacketLimit, + parse_state = ParseState, proto_state = ProtoState, - packet_opts = Env, enable_stats = EnableStats}), IdleTimout = get_value(client_idle_timeout, Env, 30000), gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, @@ -294,17 +295,17 @@ code_change(_OldVsn, State, _Extra) -> received(<<>>, State) -> {noreply, State, hibernate}; -received(Bytes, State = #client_state{parser_fun = ParserFun, - packet_opts = PacketOpts, +received(Bytes, State = #client_state{parse_state = ParseState, + packet_limit = PacketLimit, proto_state = ProtoState}) -> - case catch ParserFun(Bytes) of - {more, NewParser} -> - {noreply, run_socket(State#client_state{parser_fun = NewParser}), hibernate}; + case catch emqttd_parser:parse(Bytes, ParseState) of + {more, NewParseState} -> + {noreply, run_socket(State#client_state{parse_state = NewParseState}), hibernate}; {ok, Packet, Rest} -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - received(Rest, State#client_state{parser_fun = emqttd_parser:new(PacketOpts), + received(Rest, State#client_state{parse_state = emqttd_parser:initial_state(PacketLimit), proto_state = ProtoState1}); {error, Error} -> ?LOG(error, "Protocol error - ~p", [Error], State), diff --git a/src/emqttd_parser.erl b/src/emqttd_parser.erl index 669f4aab2..bc6e0037b 100644 --- a/src/emqttd_parser.erl +++ b/src/emqttd_parser.erl @@ -24,27 +24,22 @@ -include("emqttd_protocol.hrl"). %% API --export([new/1, parse/2]). +-export([initial_state/0, initial_state/1, parse/2]). --record(mqtt_packet_limit, {max_packet_size}). - --type(option() :: {atom(), any()}). - --type(parser() :: fun( (binary()) -> any() )). +-spec(initial_state() -> {none, pos_integer()}). +initial_state() -> + initial_state(?MAX_PACKET_LEN). %% @doc Initialize a parser --spec(new(Opts :: [option()]) -> parser()). -new(Opts) -> - fun(Bin) -> parse(Bin, {none, limit(Opts)}) end. - -limit(Opts) -> - #mqtt_packet_limit{max_packet_size = proplists:get_value(max_packet_size, Opts, ?MAX_LEN)}. +-spec(initial_state(pos_integer()) -> {none, pos_integer()}). +initial_state(MaxLen) -> + {none, MaxLen}. %% @doc Parse MQTT Packet --spec(parse(binary(), {none, [option()]} | fun()) +-spec(parse(binary(), {none, pos_integer()} | fun()) -> {ok, mqtt_packet()} | {error, any()} | {more, fun()}). -parse(<<>>, {none, Limit}) -> - {more, fun(Bin) -> parse(Bin, {none, Limit}) end}; +parse(<<>>, {none, MaxLen}) -> + {more, fun(Bin) -> parse(Bin, {none, MaxLen}) end}; parse(<>, {none, Limit}) -> parse_remaining_len(Rest, #mqtt_packet_header{type = Type, dup = bool(Dup), @@ -57,7 +52,7 @@ parse_remaining_len(<<>>, Header, Limit) -> parse_remaining_len(Rest, Header, Limit) -> parse_remaining_len(Rest, Header, 1, 0, Limit). -parse_remaining_len(_Bin, _Header, _Multiplier, Length, #mqtt_packet_limit{max_packet_size = MaxLen}) +parse_remaining_len(_Bin, _Header, _Multiplier, Length, MaxLen) when Length > MaxLen -> {error, invalid_mqtt_frame_len}; parse_remaining_len(<<>>, Header, Multiplier, Length, Limit) -> @@ -70,7 +65,7 @@ parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, _Limit) -> parse_frame(Rest, Header, 0); parse_remaining_len(<<1:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Limit) -> parse_remaining_len(Rest, Header, Multiplier * ?HIGHBIT, Value + Len * Multiplier, Limit); -parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, #mqtt_packet_limit{max_packet_size = MaxLen}) -> +parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, MaxLen) -> FrameLen = Value + Len * Multiplier, if FrameLen > MaxLen -> {error, invalid_mqtt_frame_len}; From 4d2d6fa0d6efbbb9e15e83b6770de3832fbf6f35 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 20:10:38 +0800 Subject: [PATCH 08/41] Change the default max_len to infinity --- src/emqttd_mqueue.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_mqueue.erl b/src/emqttd_mqueue.erl index 9abcc6843..4f825329a 100644 --- a/src/emqttd_mqueue.erl +++ b/src/emqttd_mqueue.erl @@ -76,7 +76,7 @@ %% priority table pseq = 0, priorities = [], %% len of simple queue - len = 0, max_len = ?MAX_LEN, + len = 0, max_len = infinity, low_wm = ?LOW_WM, high_wm = ?HIGH_WM, qos0 = false, dropped = 0, alarm_fun}). From 17953a4716c0aeb60cacd968094f300794422159 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 21 Feb 2017 20:11:09 +0800 Subject: [PATCH 09/41] Rename 'MAX_LEN' to 'MAX_PACKET_LEN' --- src/emqttd_serializer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_serializer.erl b/src/emqttd_serializer.erl index ed920f584..f6d89cc60 100644 --- a/src/emqttd_serializer.erl +++ b/src/emqttd_serializer.erl @@ -42,7 +42,7 @@ serialize_header(#mqtt_packet_header{type = Type, {VariableBin, PayloadBin}) when ?CONNECT =< Type andalso Type =< ?DISCONNECT -> Len = byte_size(VariableBin) + byte_size(PayloadBin), - true = (Len =< ?MAX_LEN), + true = (Len =< ?MAX_PACKET_LEN), [<>, serialize_len(Len), VariableBin, PayloadBin]. From 62a0eaf65d0d2c7083965d57e8a86240d4afe716 Mon Sep 17 00:00:00 2001 From: Feng Date: Tue, 21 Feb 2017 23:02:10 +0800 Subject: [PATCH 10/41] Fix the test cases for emqttd_parser --- test/emqttd_protocol_SUITE.erl | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/test/emqttd_protocol_SUITE.erl b/test/emqttd_protocol_SUITE.erl index 49189bb90..027d550a0 100644 --- a/test/emqttd_protocol_SUITE.erl +++ b/test/emqttd_protocol_SUITE.erl @@ -71,7 +71,7 @@ groups() -> %%-------------------------------------------------------------------- parse_connect(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %% CONNECT(Q0, R0, D0, ClientId=mosqpub/10451-iMac.loca, ProtoName=MQIsdp, ProtoVsn=3, CleanSess=true, KeepAlive=60, Username=undefined, Password=undefined) V31ConnBin = <<16,37,0,6,77,81,73,115,100,112,3,2,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,49,45,105,77,97,99,46,108,111,99,97>>, {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, @@ -82,7 +82,7 @@ parse_connect(_) -> proto_name = <<"MQIsdp">>, client_id = <<"mosqpub/10451-iMac.loca">>, clean_sess = true, - keep_alive = 60}}, <<>>} = Parser(V31ConnBin), + keep_alive = 60}}, <<>>} = emqttd_parser:parse(V31ConnBin, Parser), %% CONNECT(Q0, R0, D0, ClientId=mosqpub/10451-iMac.loca, ProtoName=MQTT, ProtoVsn=4, CleanSess=true, KeepAlive=60, Username=undefined, Password=undefined) V311ConnBin = <<16,35,0,4,77,81,84,84,4,2,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,49,45,105,77,97,99,46,108,111,99,97>>, {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, @@ -93,7 +93,7 @@ parse_connect(_) -> proto_name = <<"MQTT">>, client_id = <<"mosqpub/10451-iMac.loca">>, clean_sess = true, - keep_alive = 60 } }, <<>>} = Parser(V311ConnBin), + keep_alive = 60 } }, <<>>} = emqttd_parser:parse(V311ConnBin, Parser), %% CONNECT(Qos=0, Retain=false, Dup=false, ClientId="", ProtoName=MQTT, ProtoVsn=4, CleanSess=true, KeepAlive=60) V311ConnWithoutClientId = <<16,12,0,4,77,81,84,84,4,2,0,60,0,0>>, @@ -105,7 +105,7 @@ parse_connect(_) -> proto_name = <<"MQTT">>, client_id = <<>>, clean_sess = true, - keep_alive = 60 } }, <<>>} = Parser(V311ConnWithoutClientId), + keep_alive = 60 } }, <<>>} = emqttd_parser:parse(V311ConnWithoutClientId, Parser), %%CONNECT(Q0, R0, D0, ClientId=mosqpub/10452-iMac.loca, ProtoName=MQIsdp, ProtoVsn=3, CleanSess=true, KeepAlive=60, %% Username=test, Password=******, Will(Qos=1, Retain=false, Topic=/will, Msg=willmsg)) ConnBinWithWill = <<16,67,0,6,77,81,73,115,100,112,3,206,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,50,45,105,77,97,99,46,108,111,99,97,0,5,47,119,105,108,108,0,7,119,105,108,108,109,115,103,0,4,116,101,115,116,0,6,112,117,98,108,105,99>>, @@ -124,18 +124,18 @@ parse_connect(_) -> will_topic = <<"/will">>, will_msg = <<"willmsg">>, username = <<"test">>, - password = <<"public">>}}, <<>>} = Parser(ConnBinWithWill), + password = <<"public">>}}, <<>>} = emqttd_parser:parse(ConnBinWithWill, Parser), ok. parse_bridge(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), Data = <<16,86,0,6,77,81,73,115,100,112,131,44,0,60,0,19,67,95,48,48,58,48,67,58,50,57,58,50,66,58,55,55,58,53,50, 0,48,36,83,89,83,47,98,114,111,107,101,114,47,99,111,110,110,101,99,116,105,111,110,47,67,95,48,48,58,48, 67,58,50,57,58,50,66,58,55,55,58,53,50,47,115,116,97,116,101,0,1,48>>, %% CONNECT(Q0, R0, D0, ClientId=C_00:0C:29:2B:77:52, ProtoName=MQIsdp, ProtoVsn=131, CleanSess=false, KeepAlive=60, %% Username=undefined, Password=undefined, Will(Q1, R1, Topic=$SYS/broker/connection/C_00:0C:29:2B:77:52/state, Msg=0)) - {ok, #mqtt_packet{variable = Variable}, <<>>} = Parser(Data), + {ok, #mqtt_packet{variable = Variable}, <<>>} = emqttd_parser:parse(Data, Parser), #mqtt_packet_connect{client_id = <<"C_00:0C:29:2B:77:52">>, proto_ver = 16#03, proto_name = <<"MQIsdp">>, @@ -148,7 +148,7 @@ parse_bridge(_) -> will_msg = <<"0">>} = Variable. parse_publish(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %%PUBLISH(Qos=1, Retain=false, Dup=false, TopicName=a/b/c, PacketId=1, Payload=<<"hahah">>) PubBin = <<50,14,0,5,97,47,98,47,99,0,1,104,97,104,97,104>>, {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, @@ -157,7 +157,7 @@ parse_publish(_) -> retain = false}, variable = #mqtt_packet_publish{topic_name = <<"a/b/c">>, packet_id = 1}, - payload = <<"hahah">> }, <<>>} = Parser(PubBin), + payload = <<"hahah">> }, <<>>} = emqttd_parser:parse(PubBin, Parser), %PUBLISH(Qos=0, Retain=false, Dup=false, TopicName=xxx/yyy, PacketId=undefined, Payload=<<"hello">>) %DISCONNECT(Qos=0, Retain=false, Dup=false) @@ -168,43 +168,43 @@ parse_publish(_) -> retain = false}, variable = #mqtt_packet_publish{topic_name = <<"xxx/yyy">>, packet_id = undefined}, - payload = <<"hello">> }, <<224,0>>} = Parser(PubBin1), + payload = <<"hello">> }, <<224,0>>} = emqttd_parser:parse(PubBin1, Parser), {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT, dup = false, qos = 0, - retain = false}}, <<>>} = Parser(<<224, 0>>). + retain = false}}, <<>>} = emqttd_parser:parse(<<224, 0>>, Parser). parse_puback(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %%PUBACK(Qos=0, Retain=false, Dup=false, PacketId=1) {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBACK, dup = false, qos = 0, - retain = false}}, <<>>} = Parser(<<64,2,0,1>>). + retain = false}}, <<>>} = emqttd_parser:parse(<<64,2,0,1>>, Parser). parse_pubrec(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %%PUBREC(Qos=0, Retain=false, Dup=false, PacketId=1) {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREC, dup = false, qos = 0, - retain = false}}, <<>>} = Parser(<<5:4,0:4,2,0,1>>). + retain = false}}, <<>>} = emqttd_parser:parse(<<5:4,0:4,2,0,1>>, Parser). parse_pubrel(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, dup = false, qos = 1, - retain = false}}, <<>>} = Parser(<<6:4,2:4,2,0,1>>). + retain = false}}, <<>>} = emqttd_parser:parse(<<6:4,2:4,2,0,1>>, Parser). parse_pubcomp(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBCOMP, dup = false, qos = 0, - retain = false}}, <<>>} = Parser(<<7:4,0:4,2,0,1>>). + retain = false}}, <<>>} = emqttd_parser:parse(<<7:4,0:4,2,0,1>>, Parser). parse_subscribe(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %% SUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[{<<"TopicA">>,2}]) {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE, dup = false, @@ -212,10 +212,10 @@ parse_subscribe(_) -> retain = false}, variable = #mqtt_packet_subscribe{packet_id = 2, topic_table = [{<<"TopicA">>,2}]} }, <<>>} - = Parser(<<130,11,0,2,0,6,84,111,112,105,99,65,2>>). + = emqttd_parser:parse(<<130,11,0,2,0,6,84,111,112,105,99,65,2>>, Parser). parse_unsubscribe(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %% UNSUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[<<"TopicA">>]) {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE, dup = false, @@ -223,24 +223,24 @@ parse_unsubscribe(_) -> retain = false}, variable = #mqtt_packet_unsubscribe{packet_id = 2, topics = [<<"TopicA">>]}}, <<>>} - = Parser(<<162,10,0,2,0,6,84,111,112,105,99,65>>). + = emqttd_parser:parse(<<162,10,0,2,0,6,84,111,112,105,99,65>>, Parser). parse_pingreq(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PINGREQ, dup = false, qos = 0, retain = false}}, <<>>} - = Parser(<>). + = emqttd_parser:parse(<>, Parser). parse_disconnect(_) -> - Parser = emqttd_parser:new([]), + Parser = emqttd_parser:initial_state(), %DISCONNECT(Qos=0, Retain=false, Dup=false) Bin = <<224, 0>>, {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT, dup = false, qos = 0, - retain = false}}, <<>>} = Parser(Bin). + retain = false}}, <<>>} = emqttd_parser:parse(Bin, Parser). %%-------------------------------------------------------------------- %% Serialize Cases @@ -260,7 +260,7 @@ serialize_connect(_) -> serialize_connack(_) -> ConnAck = #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, variable = #mqtt_packet_connack{ack_flags = 0, return_code = 0}}, - <<32,2,0,0>> = serialize(ConnAck). + <<32,2,0,0>> = iolist_to_binary(serialize(ConnAck)). serialize_publish(_) -> serialize(?PUBLISH_PACKET(?QOS_0, <<"Topic">>, undefined, <<"Payload">>)), From d33a41b28bf0b42417bb3b3813e9f8f0178f7016 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 00:31:09 +0800 Subject: [PATCH 11/41] Remove unused fields: connname, peerhost, peerport --- src/emqttd_client.erl | 93 +++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index e58f55738..c347d73f7 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -52,16 +52,18 @@ -export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). %% Client State --record(client_state, {connection, connname, peername, peerhost, peerport, await_recv, - conn_state, rate_limit, packet_limit, parse_state, proto_state, +%% Unused fields: connname, peerhost, peerport +-record(client_state, {connection, peername, conn_state, await_recv, + rate_limit, packet_size, parser, proto_state, keepalive, enable_stats}). --define(INFO_KEYS, [connname, peername, peerhost, peerport, await_recv, conn_state]). +-define(INFO_KEYS, [peername, conn_state, await_recv]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). -define(LOG(Level, Format, Args, State), - lager:Level("Client(~s): " ++ Format, [State#client_state.connname | Args])). + lager:Level("Client(~s): " ++ Format, + [esockd_net:format(State#client_state.peername) | Args])). start_link(Conn, Env) -> {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]])}. @@ -96,50 +98,47 @@ session(CPid) -> init([Conn0, Env]) -> {ok, Conn} = Conn0:wait(), - {PeerHost, PeerPort, PeerName} = case Conn:peername() of - {ok, Peer = {Host, Port}} -> - {Host, Port, Peer}; - {error, enotconn} -> - Conn:fast_close(), - exit(normal); - {error, Reason} -> - Conn:fast_close(), - exit({shutdown, Reason}) - end, - ConnName = esockd_net:format(PeerName), + {ok, Peername} -> do_init(Conn, Env, Peername); + {error, enotconn} -> Conn:fast_close(), + exit(normal); + {error, Reason} -> Conn:fast_close(), + exit({shutdown, Reason}) + end. + +do_init(Conn, Env, Peername) -> + %% Send Fun + SendFun = send_fun(Conn, Peername), + RateLimit = get_value(rate_limit, Conn:opts()), + PacketSize = get_value(max_packet_size, Env, ?MAX_PACKET_SIZE), + Parser = emqttd_parser:initial_state(PacketSize), + ProtoState = emqttd_protocol:init(Peername, SendFun, Env), + EnableStats = get_value(client_enable_stats, Env, false), + State = run_socket(#client_state{connection = Conn, + peername = Peername, + await_recv = false, + conn_state = running, + rate_limit = RateLimit, + packet_size = PacketSize, + parser = Parser, + proto_state = ProtoState, + enable_stats = EnableStats}), + IdleTimout = get_value(client_idle_timeout, Env, 30000), + gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, + {backoff, 1000, 1000, 5000}). + +send_fun(Conn, Peername) -> Self = self(), - %% Send Packet... - SendFun = fun(Packet) -> + fun(Packet) -> Data = emqttd_serializer:serialize(Packet), - ?LOG(debug, "SEND ~p", [Data], #client_state{connname = ConnName}), + ?LOG(debug, "SEND ~p", [Data], #client_state{peername = Peername}), emqttd_metrics:inc('bytes/sent', iolist_size(Data)), try Conn:async_send(Data) of true -> ok catch error:Error -> Self ! {shutdown, Error} end - end, - RateLimit = get_value(rate_limit, Conn:opts()), - PacketLimit = proplists:get_value(max_packet_size, Env, ?MAX_PACKET_LEN), - ParseState = emqttd_parser:initial_state(PacketLimit), - ProtoState = emqttd_protocol:init(PeerName, SendFun, Env), - EnableStats = get_value(client_enable_stats, Env, false), - State = run_socket(#client_state{connection = Conn, - connname = ConnName, - peername = PeerName, - peerhost = PeerHost, - peerport = PeerPort, - await_recv = false, - conn_state = running, - rate_limit = RateLimit, - packet_limit = PacketLimit, - parse_state = ParseState, - proto_state = ProtoState, - enable_stats = EnableStats}), - IdleTimout = get_value(client_idle_timeout, Env, 30000), - gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, - {backoff, 1000, 1000, 5000}). + end. prioritise_call(Msg, _From, _Len, _State) -> case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. @@ -147,8 +146,8 @@ prioritise_call(Msg, _From, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. -handle_pre_hibernate(State = #client_state{connname = Connname}) -> - io:format("Client(~s) will hibernate!~n", [Connname]), +handle_pre_hibernate(State = #client_state{peername = Peername}) -> + io:format("Client(~s) will hibernate!~n", [esockd_net:format(Peername)]), {hibernate, emit_stats(State)}. handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> @@ -295,17 +294,17 @@ code_change(_OldVsn, State, _Extra) -> received(<<>>, State) -> {noreply, State, hibernate}; -received(Bytes, State = #client_state{parse_state = ParseState, - packet_limit = PacketLimit, +received(Bytes, State = #client_state{parser = Parser, + packet_size = PacketSize, proto_state = ProtoState}) -> - case catch emqttd_parser:parse(Bytes, ParseState) of - {more, NewParseState} -> - {noreply, run_socket(State#client_state{parse_state = NewParseState}), hibernate}; + case catch emqttd_parser:parse(Bytes, Parser) of + {more, NewParser} -> + {noreply, run_socket(State#client_state{parser = NewParser}), hibernate}; {ok, Packet, Rest} -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - received(Rest, State#client_state{parse_state = emqttd_parser:initial_state(PacketLimit), + received(Rest, State#client_state{parser = emqttd_parser:initial_state(PacketSize), proto_state = ProtoState1}); {error, Error} -> ?LOG(error, "Protocol error - ~p", [Error], State), From 418fb37ca0f32dc179c8ff4fea010edea8bba497 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 00:31:47 +0800 Subject: [PATCH 12/41] Add max_packet_size() type --- src/emqttd_parser.erl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/emqttd_parser.erl b/src/emqttd_parser.erl index bc6e0037b..dde9ae4dc 100644 --- a/src/emqttd_parser.erl +++ b/src/emqttd_parser.erl @@ -26,14 +26,16 @@ %% API -export([initial_state/0, initial_state/1, parse/2]). --spec(initial_state() -> {none, pos_integer()}). +-type(max_packet_size() :: 1..?MAX_PACKET_SIZE). + +-spec(initial_state() -> {none, max_packet_size()}). initial_state() -> - initial_state(?MAX_PACKET_LEN). + initial_state(?MAX_PACKET_SIZE). %% @doc Initialize a parser --spec(initial_state(pos_integer()) -> {none, pos_integer()}). -initial_state(MaxLen) -> - {none, MaxLen}. +-spec(initial_state(max_packet_size()) -> {none, max_packet_size()}). +initial_state(MaxSize) -> + {none, MaxSize}. %% @doc Parse MQTT Packet -spec(parse(binary(), {none, pos_integer()} | fun()) From 6c50a59cadc949bb4ed1708fb8efaece4c1d4d19 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 00:32:22 +0800 Subject: [PATCH 13/41] Rename macro 'MAX_PACKET_LEN' to 'MAX_PACKET_SIZE' --- include/emqttd_protocol.hrl | 2 +- src/emqttd_serializer.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/emqttd_protocol.hrl b/include/emqttd_protocol.hrl index a7eb768e3..6181b9a65 100644 --- a/include/emqttd_protocol.hrl +++ b/include/emqttd_protocol.hrl @@ -133,7 +133,7 @@ %% Max MQTT Packet Length %%-------------------------------------------------------------------- --define(MAX_PACKET_LEN, 16#fffffff). +-define(MAX_PACKET_SIZE, 16#fffffff). %%-------------------------------------------------------------------- %% MQTT Parser and Serializer diff --git a/src/emqttd_serializer.erl b/src/emqttd_serializer.erl index f6d89cc60..a47a23c8b 100644 --- a/src/emqttd_serializer.erl +++ b/src/emqttd_serializer.erl @@ -42,7 +42,7 @@ serialize_header(#mqtt_packet_header{type = Type, {VariableBin, PayloadBin}) when ?CONNECT =< Type andalso Type =< ?DISCONNECT -> Len = byte_size(VariableBin) + byte_size(PayloadBin), - true = (Len =< ?MAX_PACKET_LEN), + true = (Len =< ?MAX_PACKET_SIZE), [<>, serialize_len(Len), VariableBin, PayloadBin]. From 700ec7aaefe6e63c2254fbd186cdf8c060f75b2b Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 10:01:39 +0800 Subject: [PATCH 14/41] Add 'proto_stats' record --- src/emqttd_protocol.erl | 54 ++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index da235a44e..7ee9a7a6a 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -35,12 +35,14 @@ -export([process/2]). +-record(proto_stats, {recv_pkt = 0, recv_msg = 0, send_pkt = 0, send_msg = 0}). + %% Protocol State -record(proto_state, {peername, sendfun, connected = false, client_id, client_pid, clean_sess, proto_ver, proto_name, username, is_superuser = false, will_msg, keepalive, max_clientid_len = ?MAX_CLIENTID_LEN, - session, ws_initial_headers, %% Headers from first HTTP request for websocket client + session, stats, ws_initial_headers, %% Headers from first HTTP request for websocket client connected_at}). -type(proto_state() :: #proto_state{}). @@ -56,20 +58,20 @@ %% @doc Init protocol init(Peername, SendFun, Opts) -> - lists:foreach(fun(K) -> put(K, 0) end, ?STATS_KEYS), MaxLen = get_value(max_clientid_len, Opts, ?MAX_CLIENTID_LEN), WsInitialHeaders = get_value(ws_initial_headers, Opts), #proto_state{peername = Peername, sendfun = SendFun, max_clientid_len = MaxLen, client_pid = self(), + stats = #proto_stats{}, ws_initial_headers = WsInitialHeaders}. info(ProtoState) -> ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS). -stats(_ProtoState) -> - [{K, get(K)} || K <- ?STATS_KEYS]. +stats(#proto_state{stats = Stats}) -> + ?record_to_proplist(proto_stats, Stats). clientid(#proto_state{client_id = ClientId}) -> ClientId. @@ -106,8 +108,10 @@ session(#proto_state{session = Session}) -> %% A Client can only send the CONNECT Packet once over a Network Connection. -spec(received(mqtt_packet(), proto_state()) -> {ok, proto_state()} | {error, any()}). -received(Packet = ?PACKET(?CONNECT), State = #proto_state{connected = false}) -> - process(Packet, State#proto_state{connected = true}); +received(Packet = ?PACKET(?CONNECT), + State = #proto_state{connected = false, stats = Stats}) -> + trace(recv, Packet, State), Stats1 = inc_stats(recv, ?CONNECT, Stats), + process(Packet, State#proto_state{connected = true, stats = Stats1}); received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> {error, protocol_bad_connect, State}; @@ -116,11 +120,11 @@ received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> received(_Packet, State = #proto_state{connected = false}) -> {error, protocol_not_connected, State}; -received(Packet = ?PACKET(_Type), State) -> - trace(recv, Packet, State), +received(Packet = ?PACKET(Type), State = #proto_state{stats = Stats}) -> + trace(recv, Packet, State), Stats1 = inc_stats(recv, Type, Stats), case validate_packet(Packet) of ok -> - process(Packet, State); + process(Packet, State#proto_state{stats = Stats1}); {error, Reason} -> {error, Reason, State} end. @@ -151,7 +155,7 @@ unsubscribe(RawTopics, ProtoState = #proto_state{client_id = ClientId, %% @doc Send PUBREL pubrel(PacketId, State) -> send(?PUBREL_PACKET(PacketId), State). -process(Packet = ?CONNECT_PACKET(Var), State0) -> +process(?CONNECT_PACKET(Var), State0) -> #mqtt_packet_connect{proto_ver = ProtoVer, proto_name = ProtoName, @@ -170,8 +174,6 @@ process(Packet = ?CONNECT_PACKET(Var), State0) -> will_msg = willmsg(Var), connected_at = os:timestamp()}, - trace(recv, Packet, State1), - {ReturnCode1, SessPresent, State3} = case validate_connect(Var, State1) of ?CONNACK_ACCEPT -> @@ -312,22 +314,34 @@ send(Msg, State = #proto_state{client_id = ClientId, username = Username}) emqttd_hooks:run('message.delivered', [ClientId, Username], Msg), send(emqttd_message:to_packet(Msg), State); -send(Packet, State = #proto_state{sendfun = SendFun}) - when is_record(Packet, mqtt_packet) -> +send(Packet = ?PACKET(Type), + State = #proto_state{sendfun = SendFun, stats = Stats}) -> trace(send, Packet, State), emqttd_metrics:sent(Packet), SendFun(Packet), - {ok, State}. + Stats1 = inc_stats(send, Type, Stats), + {ok, State#proto_state{stats = Stats1}}. -trace(recv, Packet = ?PACKET(Type), ProtoState) -> - inc(recv_pkt), ?IF(Type =:= ?PUBLISH, inc(recv_msg), ok), +trace(recv, Packet, ProtoState) -> ?LOG(info, "RECV ~s", [emqttd_packet:format(Packet)], ProtoState); -trace(send, Packet = ?PACKET(Type), ProtoState) -> - inc(send_pkt), ?IF(Type =:= ?PUBLISH, inc(send_msg), ok), +trace(send, Packet, ProtoState) -> ?LOG(info, "SEND ~s", [emqttd_packet:format(Packet)], ProtoState). -inc(Key) -> put(Key, get(Key) + 1). +inc_stats(recv, Type, Stats) -> + #proto_stats{recv_pkt = Pkt, recv_msg = Msg} = Stats, + inc_stats(Type, #proto_stats.recv_pkt, Pkt, #proto_stats.recv_msg, Msg, Stats); + +inc_stats(send, Type, Stats) -> + #proto_stats{send_pkt = Pkt, send_msg = Msg} = Stats, + inc_stats(Type, #proto_stats.send_pkt, Pkt, #proto_stats.send_msg, Msg, Stats). + +inc_stats(Type, PktPos, PktCnt, MsgPos, MsgCnt, Stats) -> + Stats1 = setelement(PktPos, Stats, PktCnt + 1), + case Type =:= ?PUBLISH of + true -> setelement(MsgPos, Stats1, MsgCnt + 1); + false -> Stats1 + end. stop_if_auth_failure(RC, State) when RC == ?CONNACK_CREDENTIALS; RC == ?CONNACK_AUTH -> {stop, {shutdown, auth_failure}, State}; From f4c4e5635c31c5c017ac8d3c1ee37f98cc8856b5 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 12:10:52 +0800 Subject: [PATCH 15/41] Improve the 'enable_stats' design of client, session --- src/emqttd_client.erl | 6 +-- src/emqttd_protocol.erl | 41 +++++++++-------- src/emqttd_ws_client.erl | 95 +++++++++++++++++++++------------------- 3 files changed, 77 insertions(+), 65 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index c347d73f7..93efa938b 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc MQTT/TCP Connection +%% @doc MQTT/TCP Connection. -module(emqttd_client). @@ -57,7 +57,7 @@ rate_limit, packet_size, parser, proto_state, keepalive, enable_stats}). --define(INFO_KEYS, [peername, conn_state, await_recv]). +-define(INFO_KEYS, [peername, conn_state, await_recv, enable_stats]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -125,7 +125,7 @@ do_init(Conn, Env, Peername) -> enable_stats = EnableStats}), IdleTimout = get_value(client_idle_timeout, Env, 30000), gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, - {backoff, 1000, 1000, 5000}). + {backoff, 1000, 1000, 10000}). send_fun(Conn, Peername) -> Self = self(), diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index 7ee9a7a6a..4b3c3ca76 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -35,15 +35,15 @@ -export([process/2]). --record(proto_stats, {recv_pkt = 0, recv_msg = 0, send_pkt = 0, send_msg = 0}). +-record(proto_stats, {enable_stats = false, recv_pkt = 0, recv_msg = 0, + send_pkt = 0, send_msg = 0}). %% Protocol State --record(proto_state, {peername, sendfun, connected = false, - client_id, client_pid, clean_sess, - proto_ver, proto_name, username, is_superuser = false, - will_msg, keepalive, max_clientid_len = ?MAX_CLIENTID_LEN, - session, stats, ws_initial_headers, %% Headers from first HTTP request for websocket client - connected_at}). +%% ws_initial_headers: Headers from first HTTP request for WebSocket Client. +-record(proto_state, {peername, sendfun, connected = false, client_id, client_pid, + clean_sess, proto_ver, proto_name, username, is_superuser, + will_msg, keepalive, max_clientid_len, session, stats_data, + ws_initial_headers, connected_at}). -type(proto_state() :: #proto_state{}). @@ -58,20 +58,22 @@ %% @doc Init protocol init(Peername, SendFun, Opts) -> + EnableStats = get_value(client_enable_stats, Opts, false), MaxLen = get_value(max_clientid_len, Opts, ?MAX_CLIENTID_LEN), WsInitialHeaders = get_value(ws_initial_headers, Opts), #proto_state{peername = Peername, sendfun = SendFun, - max_clientid_len = MaxLen, client_pid = self(), - stats = #proto_stats{}, - ws_initial_headers = WsInitialHeaders}. + max_clientid_len = MaxLen, + is_superuser = false, + ws_initial_headers = WsInitialHeaders, + stats_data = #proto_stats{enable_stats = EnableStats}}. info(ProtoState) -> ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS). -stats(#proto_state{stats = Stats}) -> - ?record_to_proplist(proto_stats, Stats). +stats(#proto_state{stats_data = Stats}) -> + tl(?record_to_proplist(proto_stats, Stats)). clientid(#proto_state{client_id = ClientId}) -> ClientId. @@ -109,9 +111,9 @@ session(#proto_state{session = Session}) -> %% A Client can only send the CONNECT Packet once over a Network Connection. -spec(received(mqtt_packet(), proto_state()) -> {ok, proto_state()} | {error, any()}). received(Packet = ?PACKET(?CONNECT), - State = #proto_state{connected = false, stats = Stats}) -> + State = #proto_state{connected = false, stats_data = Stats}) -> trace(recv, Packet, State), Stats1 = inc_stats(recv, ?CONNECT, Stats), - process(Packet, State#proto_state{connected = true, stats = Stats1}); + process(Packet, State#proto_state{connected = true, stats_data = Stats1}); received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> {error, protocol_bad_connect, State}; @@ -120,11 +122,11 @@ received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> received(_Packet, State = #proto_state{connected = false}) -> {error, protocol_not_connected, State}; -received(Packet = ?PACKET(Type), State = #proto_state{stats = Stats}) -> +received(Packet = ?PACKET(Type), State = #proto_state{stats_data = Stats}) -> trace(recv, Packet, State), Stats1 = inc_stats(recv, Type, Stats), case validate_packet(Packet) of ok -> - process(Packet, State#proto_state{stats = Stats1}); + process(Packet, State#proto_state{stats_data = Stats1}); {error, Reason} -> {error, Reason, State} end. @@ -315,12 +317,12 @@ send(Msg, State = #proto_state{client_id = ClientId, username = Username}) send(emqttd_message:to_packet(Msg), State); send(Packet = ?PACKET(Type), - State = #proto_state{sendfun = SendFun, stats = Stats}) -> + State = #proto_state{sendfun = SendFun, stats_data = Stats}) -> trace(send, Packet, State), emqttd_metrics:sent(Packet), SendFun(Packet), Stats1 = inc_stats(send, Type, Stats), - {ok, State#proto_state{stats = Stats1}}. + {ok, State#proto_state{stats_data = Stats1}}. trace(recv, Packet, ProtoState) -> ?LOG(info, "RECV ~s", [emqttd_packet:format(Packet)], ProtoState); @@ -328,6 +330,9 @@ trace(recv, Packet, ProtoState) -> trace(send, Packet, ProtoState) -> ?LOG(info, "SEND ~s", [emqttd_packet:format(Packet)], ProtoState). +inc_stats(_Direct, _Type, Stats = #proto_stats{enable_stats = false}) -> + Stats; + inc_stats(recv, Type, Stats) -> #proto_stats{recv_pkt = Pkt, recv_msg = Msg} = Stats, inc_stats(Type, #proto_stats.recv_pkt, Pkt, #proto_stats.recv_msg, Msg, Stats); diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 71cb4b344..c6ea5c1ad 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -14,6 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% @doc MQTT WebSocket Connection. + -module(emqttd_ws_client). -behaviour(gen_server2). @@ -24,6 +26,8 @@ -include("emqttd_protocol.hrl"). +-import(proplists, [get_value/3]). + %% API Exports -export([start_link/4]). @@ -44,13 +48,14 @@ -export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). %% WebSocket Client State --record(wsclient_state, {ws_pid, peer, connection, proto_state, keepalive, +-record(wsclient_state, {ws_pid, peername, connection, proto_state, keepalive, enable_stats}). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). --define(WSLOG(Level, Peer, Format, Args), - lager:Level("WsClient(~s): " ++ Format, [Peer | Args])). +-define(WSLOG(Level, Format, Args, State), + lager:Level("WsClient(~s): " ++ Format, + [esockd_net:format(State#wsclient_state.peername) | Args])). %% @doc Start WebSocket Client. start_link(Env, WsPid, Req, ReplyChannel) -> @@ -84,22 +89,16 @@ init([Env, WsPid, Req, ReplyChannel]) -> {ok, Peername} = Req:get(peername), Headers = mochiweb_headers:to_list( mochiweb_request:get(headers, Req)), - %% SendFun = fun(Payload) -> ReplyChannel({binary, Payload}) end, - SendFun = fun(Packet) -> - Data = emqttd_serializer:serialize(Packet), - emqttd_metrics:inc('bytes/sent', iolist_size(Data)), - ReplyChannel({binary, Data}) - end, - EnableStats = proplists:get_value(client_enable_stats, Env, false), - ProtoState = emqttd_protocol:init(Peername, SendFun, + ProtoState = emqttd_protocol:init(Peername, send_fun(ReplyChannel), [{ws_initial_headers, Headers} | Env]), - IdleTimeout = proplists:get_value(client_idle_timeout, Env, 30000), + IdleTimeout = get_value(client_idle_timeout, Env, 30000), + EnableStats = get_value(client_enable_stats, Env, false), {ok, #wsclient_state{ws_pid = WsPid, - peer = Req:get(peer), + peername = Peername, connection = Req:get(connection), proto_state = ProtoState, enable_stats = EnableStats}, - IdleTimeout, {backoff, 1000, 1000, 5000}, ?MODULE}. + IdleTimeout, {backoff, 1000, 1000, 10000}, ?MODULE}. prioritise_call(Msg, _From, _Len, _State) -> case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. @@ -107,12 +106,12 @@ prioritise_call(Msg, _From, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. -handle_pre_hibernate(State = #wsclient_state{peer = Peer}) -> - io:format("WsClient(~s) will hibernate!~n", [Peer]), +handle_pre_hibernate(State) -> {hibernate, emit_stats(State)}. -handle_call(info, From, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) -> - Info = [{websocket, true}, {peer, Peer} | emqttd_protocol:info(ProtoState)], +handle_call(info, From, State = #wsclient_state{peername = Peername, + proto_state = ProtoState}) -> + Info = [{websocket, true}, {peername, Peername} | emqttd_protocol:info(ProtoState)], {reply, Stats, _, _} = handle_call(stats, From, State), reply(lists:append(Info, Stats), State); @@ -127,17 +126,17 @@ handle_call(kick, _From, State) -> handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) -> reply(emqttd_protocol:session(ProtoState), State); -handle_call(Req, _From, State = #wsclient_state{peer = Peer}) -> - ?WSLOG(error, Peer, "Unexpected request: ~p", [Req]), - reply({error, unsupported_request}, State). +handle_call(Req, _From, State) -> + ?WSLOG(error, "Unexpected request: ~p", [Req], State), + reply({error, unexpected_request}, State). -handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state = ProtoState}) -> +handle_cast({received, Packet}, State = #wsclient_state{proto_state = ProtoState}) -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> {noreply, State#wsclient_state{proto_state = ProtoState1}, hibernate}; {error, Error} -> - ?WSLOG(error, Peer, "Protocol error - ~p", [Error]), + ?WSLOG(error, "Protocol error - ~p", [Error], State), shutdown(Error, State); {error, Error, ProtoState1} -> shutdown(Error, State#wsclient_state{proto_state = ProtoState1}); @@ -145,8 +144,8 @@ handle_cast({received, Packet}, State = #wsclient_state{peer = Peer, proto_state stop(Reason, State#wsclient_state{proto_state = ProtoState1}) end; -handle_cast(Msg, State = #wsclient_state{peer = Peer}) -> - ?WSLOG(error, Peer, "Unexpected msg: ~p", [Msg]), +handle_cast(Msg, State) -> + ?WSLOG(error, "Unexpected Msg: ~p", [Msg], State), {noreply, State, hibernate}. handle_info({subscribe, TopicTable}, State) -> @@ -186,43 +185,36 @@ handle_info(emit_stats, State) -> handle_info(timeout, State) -> shutdown(idle_timeout, State); -handle_info({shutdown, conflict, {ClientId, NewPid}}, State = #wsclient_state{peer = Peer}) -> - ?WSLOG(warning, Peer, "clientid '~s' conflict with ~p", [ClientId, NewPid]), +handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> + ?WSLOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State), shutdown(conflict, State); -handle_info({keepalive, start, Interval}, State = #wsclient_state{peer = Peer, connection = Conn}) -> - ?WSLOG(debug, Peer, "Keepalive at the interval of ~p", [Interval]), - StatFun = fun() -> - case Conn:getstat([recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; - {error, Error} -> {error, Error} - end - end, - KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}), +handle_info({keepalive, start, Interval}, State = #wsclient_state{connection = Conn}) -> + ?WSLOG(debug, "Keepalive at the interval of ~p", [Interval], State), + KeepAlive = emqttd_keepalive:start(stat_fun(Conn), Interval, {keepalive, check}), {noreply, State#wsclient_state{keepalive = KeepAlive}, hibernate}; -handle_info({keepalive, check}, State = #wsclient_state{peer = Peer, - keepalive = KeepAlive}) -> +handle_info({keepalive, check}, State = #wsclient_state{keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> {noreply, emit_stats(State#wsclient_state{keepalive = KeepAlive1}), hibernate}; {error, timeout} -> - ?WSLOG(debug, Peer, "Keepalive Timeout!", []), + ?WSLOG(debug, "Keepalive Timeout!", [], State), shutdown(keepalive_timeout, State); {error, Error} -> - ?WSLOG(warning, Peer, "Keepalive error - ~p", [Error]), + ?WSLOG(warning, "Keepalive error - ~p", [Error], State), shutdown(keepalive_error, State) end; handle_info({'EXIT', WsPid, normal}, State = #wsclient_state{ws_pid = WsPid}) -> stop(normal, State); -handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{peer = Peer, ws_pid = WsPid}) -> - ?WSLOG(error, Peer, "shutdown: ~p",[Reason]), +handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{ws_pid = WsPid}) -> + ?WSLOG(error, "shutdown: ~p",[Reason], State), shutdown(Reason, State); -handle_info(Info, State = #wsclient_state{peer = Peer}) -> - ?WSLOG(error, Peer, "Unexpected Info: ~p", [Info]), +handle_info(Info, State) -> + ?WSLOG(error, "Unexpected Info: ~p", [Info], State), {noreply, State, hibernate}. terminate(Reason, #wsclient_state{proto_state = ProtoState, keepalive = KeepAlive}) -> @@ -241,6 +233,21 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- +send_fun(ReplyChannel) -> + fun(Packet) -> + Data = emqttd_serializer:serialize(Packet), + emqttd_metrics:inc('bytes/sent', iolist_size(Data)), + ReplyChannel({binary, Data}) + end. + +stat_fun(Conn) -> + fun() -> + case Conn:getstat([recv_oct]) of + {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; + {error, Error} -> {error, Error} + end + end. + emit_stats(State = #wsclient_state{proto_state = ProtoState}) -> emit_stats(emqttd_protocol:clientid(ProtoState), State). From 088adeda3bcb76e27121c60932c59c5e7bbedd9d Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 14:11:10 +0800 Subject: [PATCH 16/41] Fix issue #916 - add 'mqtt_msg_from()' type --- include/emqttd.hrl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/emqttd.hrl b/include/emqttd.hrl index 0d7d4c134..3f4f22229 100644 --- a/include/emqttd.hrl +++ b/include/emqttd.hrl @@ -108,13 +108,15 @@ -type(mqtt_pktid() :: 1..16#ffff | undefined). +-type(mqtt_msg_from() :: atom() | {binary(), undefined | binary()}). + -record(mqtt_message, { %% Global unique message ID id :: mqtt_msgid(), %% PacketId pktid :: mqtt_pktid(), %% ClientId and Username - from :: {binary(), undefined | binary()}, + from :: mqtt_msg_from(), %% Topic that the message is published to topic :: binary(), %% Message QoS @@ -127,12 +129,13 @@ dup = false :: boolean(), %% $SYS flag sys = false :: boolean(), + %% Headers headers = [] :: list(), %% Payload payload :: binary(), %% Timestamp timestamp :: erlang:timestamp() -}). + }). -type(mqtt_message() :: #mqtt_message{}). From b4936726fd368ecf911ef8ac519ee2b2e3e99eff Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 15:36:06 +0800 Subject: [PATCH 17/41] Remove the random:seed/1 to fix the build warnings --- src/emqttd_time.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/emqttd_time.erl b/src/emqttd_time.erl index f9ada795d..7e5940438 100644 --- a/src/emqttd_time.erl +++ b/src/emqttd_time.erl @@ -21,10 +21,7 @@ -export([seed/0, now_secs/0, now_secs/1, now_ms/0, now_ms/1, ts_from_ms/1]). seed() -> - case erlang:function_exported(erlang, timestamp, 0) of - true -> rand:seed(exsplus, erlang:timestamp()); %% R18 - false -> random:seed(os:timestamp()) %% Compress now() deprecated warning... - end. + rand:seed(exsplus, erlang:timestamp()). now_ms() -> now_ms(os:timestamp()). @@ -40,3 +37,4 @@ now_secs({MegaSecs, Secs, _MicroSecs}) -> ts_from_ms(Ms) -> {Ms div 1000000, Ms rem 1000000, 0}. + From 88c2b4eaa3cfe2d523fbe896aaaec66dd8484980 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 15:43:24 +0800 Subject: [PATCH 18/41] Use the new emqttd_parser API to parse Websocket frame --- src/emqttd_ws.erl | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/emqttd_ws.erl b/src/emqttd_ws.erl index 8c483344e..b292c39bc 100644 --- a/src/emqttd_ws.erl +++ b/src/emqttd_ws.erl @@ -18,13 +18,18 @@ -author("Feng Lee "). +-include("emqttd_protocol.hrl"). + +-import(proplists, [get_value/3]). + -export([handle_request/1, ws_loop/3]). %% WebSocket Loop State --record(wsocket_state, {peer, client_pid, packet_opts, parser_fun}). +-record(wsocket_state, {peername, client_pid, max_packet_size, parser}). --define(WSLOG(Level, Peer, Format, Args), - lager:Level("WsClient(~s): " ++ Format, [Peer | Args])). +-define(WSLOG(Level, Format, Args, State), + lager:Level("WsClient(~s): " ++ Format, + [esockd_net:format(State#wsocket_state.peername) | Args])). %%-------------------------------------------------------------------- %% Handle WebSocket Request @@ -32,18 +37,14 @@ %% @doc Handle WebSocket Request. handle_request(Req) -> - Peer = Req:get(peer), - {ok, PktOpts} = emqttd:env(protocol), - ParserFun = emqttd_parser:new(PktOpts), - {ReentryWs, ReplyChannel} = upgrade(Req), + {ok, Env} = emqttd:env(protocol), + PacketSize = get_value(max_packet_size, Env, ?MAX_PACKET_SIZE), + Parser = emqttd_parser:initial_state(PacketSize), + %% Upgrade WebSocket. + {ReentryWs, ReplyChannel} = mochiweb_websocket:upgrade_connection(Req, fun ?MODULE:ws_loop/3), {ok, ClientPid} = emqttd_ws_client_sup:start_client(self(), Req, ReplyChannel), - ReentryWs(#wsocket_state{peer = Peer, client_pid = ClientPid, - packet_opts = PktOpts, parser_fun = ParserFun}). - -%% @doc Upgrade WebSocket. -%% @private -upgrade(Req) -> - mochiweb_websocket:upgrade_connection(Req, fun ?MODULE:ws_loop/3). + ReentryWs(#wsocket_state{peername = Req:get(peername), parser = Parser, + max_packet_size = PacketSize, client_pid = ClientPid}). %%-------------------------------------------------------------------- %% Receive Loop @@ -54,25 +55,24 @@ ws_loop(<<>>, State, _ReplyChannel) -> State; ws_loop([<<>>], State, _ReplyChannel) -> State; -ws_loop(Data, State = #wsocket_state{peer = Peer, client_pid = ClientPid, - parser_fun = ParserFun}, ReplyChannel) -> - ?WSLOG(debug, Peer, "RECV ~p", [Data]), +ws_loop(Data, State = #wsocket_state{client_pid = ClientPid, parser = Parser}, ReplyChannel) -> + ?WSLOG(debug, "RECV ~p", [Data], State), emqttd_metrics:inc('bytes/received', iolist_size(Data)), - case catch ParserFun(iolist_to_binary(Data)) of + case catch emqttd_parser:parse(iolist_to_binary(Data), Parser) of {more, NewParser} -> - State#wsocket_state{parser_fun = NewParser}; + State#wsocket_state{parser = NewParser}; {ok, Packet, Rest} -> gen_server:cast(ClientPid, {received, Packet}), ws_loop(Rest, reset_parser(State), ReplyChannel); {error, Error} -> - ?WSLOG(error, Peer, "Frame error: ~p", [Error]), + ?WSLOG(error, "Frame error: ~p", [Error], State), exit({shutdown, Error}); {'EXIT', Reason} -> - ?WSLOG(error, Peer, "Frame error: ~p", [Reason]), - ?WSLOG(error, Peer, "Error data: ~p", [Data]), + ?WSLOG(error, "Frame error: ~p", [Reason], State), + ?WSLOG(error, "Error data: ~p", [Data], State), exit({shutdown, parser_error}) end. -reset_parser(State = #wsocket_state{packet_opts = PktOpts}) -> - State#wsocket_state{parser_fun = emqttd_parser:new(PktOpts)}. +reset_parser(State = #wsocket_state{max_packet_size = PacketSize}) -> + State#wsocket_state{parser = emqttd_parser:initial_state(PacketSize)}. From 7d16422d5c2417f5b16baa1587b7ecdc5daf2ef8 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 15:56:12 +0800 Subject: [PATCH 19/41] Require R19+ to build since 2.1 release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dc7ea803..c2bbf5ebd 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Documentation on [emqtt.io/docs/v2/](http://emqtt.io/docs/v2/install.html), [doc ## Build From Source -The *EMQ* broker requires Erlang/OTP R18+ to build. +The *EMQ* broker requires Erlang/OTP R19+ to build since 2.1 release. ``` git clone https://github.com/emqtt/emq-relx.git From 05e34fe78314c816537f1542157e89999e5f7ea1 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 16:19:49 +0800 Subject: [PATCH 20/41] Move the 'cuttlefish' library from TEST_DEPS to BUILD_DEPS --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1447dd30c..eee4adc4d 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ PROJECT = emqttd PROJECT_DESCRIPTION = Erlang MQTT Broker PROJECT_VERSION = 2.1 +NO_AUTOPATCH = cuttlefish + DEPS = gproc lager esockd mochiweb lager_syslog dep_gproc = git https://github.com/uwiger/gproc @@ -13,11 +15,11 @@ dep_lager_syslog = git https://github.com/basho/lager_syslog ERLC_OPTS += +'{parse_transform, lager_transform}' -TEST_DEPS = cuttlefish emqttc +BUILD_DEPS = cuttlefish dep_cuttlefish = git https://github.com/emqtt/cuttlefish -dep_emqttc = git https://github.com/emqtt/emqttc -NO_AUTOPATCH = cuttlefish +TEST_DEPS = emqttc +dep_emqttc = git https://github.com/emqtt/emqttc TEST_ERLC_OPTS += +debug_info TEST_ERLC_OPTS += +'{parse_transform, lager_transform}' @@ -38,5 +40,5 @@ include erlang.mk app:: rebar.config app.config:: - cuttlefish -l info -e etc/ -c etc/emq.conf -i priv/emq.schema -d data/ + ./deps/cuttlefish/cuttlefish -l info -e etc/ -c etc/emq.conf -i priv/emq.schema -d data/ From 03e6c8b64cd257e7fd5b90d28891b34405e7d139 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 17:33:25 +0800 Subject: [PATCH 21/41] Version 2.1.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eee4adc4d..913f11b89 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROJECT = emqttd PROJECT_DESCRIPTION = Erlang MQTT Broker -PROJECT_VERSION = 2.1 +PROJECT_VERSION = 2.1.0 NO_AUTOPATCH = cuttlefish From 8e7ea09cbd2a026bfab7c47ac9efbd983257e2e2 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 17:34:00 +0800 Subject: [PATCH 22/41] Tune the mqtt.client|session.enable_stats off --- etc/emq.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/emq.conf b/etc/emq.conf index 0e20dd60d..f042bd279 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -114,7 +114,7 @@ mqtt.max_packet_size = 64KB mqtt.client.idle_timeout = 30s ## Enable client Stats: on | off -mqtt.client.enable_stats = on +mqtt.client.enable_stats = off ##-------------------------------------------------------------------- ## MQTT Session @@ -137,7 +137,7 @@ mqtt.session.max_awaiting_rel = 100 mqtt.session.await_rel_timeout = 20s ## Enable Statistics: on | off -mqtt.session.enable_stats = on +mqtt.session.enable_stats = off ## Expired after 1 day: ## w - week From 39abdb8b41685e7fb714e64e5bb9fe0f6446cbcf Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 17:52:55 +0800 Subject: [PATCH 23/41] Remove 'enable_stats' from 'INFO_KEYS', and remove io:format line --- src/emqttd_client.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 93efa938b..af36418d9 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -57,7 +57,7 @@ rate_limit, packet_size, parser, proto_state, keepalive, enable_stats}). --define(INFO_KEYS, [peername, conn_state, await_recv, enable_stats]). +-define(INFO_KEYS, [peername, conn_state, await_recv]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -146,8 +146,7 @@ prioritise_call(Msg, _From, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. -handle_pre_hibernate(State = #client_state{peername = Peername}) -> - io:format("Client(~s) will hibernate!~n", [esockd_net:format(Peername)]), +handle_pre_hibernate(State) -> {hibernate, emit_stats(State)}. handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> From f955614f9dbf173f9c3a4d67abc5f0ab34916e5d Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 18:40:17 +0800 Subject: [PATCH 24/41] Remove the io:format line --- src/emqttd_session.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 852f46c5f..2a27f2906 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -333,8 +333,7 @@ prioritise_info(Msg, _Len, _State) -> _ -> 0 end. -handle_pre_hibernate(State = #state{client_id = ClientId}) -> - io:format("Session(~s) will hibernate!~n", [ClientId]), +handle_pre_hibernate(State) -> {hibernate, emit_stats(State)}. handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, @@ -539,8 +538,7 @@ handle_info({timeout, _Timer, expired}, State) -> shutdown(expired, State); handle_info({'EXIT', ClientPid, _Reason}, - State = #state{clean_sess = true, - client_pid = ClientPid}) -> + State = #state{clean_sess = true, client_pid = ClientPid}) -> {stop, normal, State}; handle_info({'EXIT', ClientPid, Reason}, From 5543b7224315a7c9c903b5e63c9a33441b3c8451 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 18:41:02 +0800 Subject: [PATCH 25/41] Force to GC the Websocket Process --- src/emqttd_ws_client.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index c6ea5c1ad..c7cff7cea 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -106,7 +106,8 @@ prioritise_call(Msg, _From, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. -handle_pre_hibernate(State) -> +handle_pre_hibernate(State = #wsclient_state{ws_pid = WsPid}) -> + erlang:garbage_collect(WsPid),%%TODO: [{async, RequestId}]?? {hibernate, emit_stats(State)}. handle_call(info, From, State = #wsclient_state{peername = Peername, From 5d5de51f89e3c964a89992431ff6c277183c2f0b Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 22 Feb 2017 18:47:14 +0800 Subject: [PATCH 26/41] Change the gen_server:start_link to gen_server2 --- src/emqttd_ws_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index c7cff7cea..b4c9d4fe9 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -59,7 +59,7 @@ %% @doc Start WebSocket Client. start_link(Env, WsPid, Req, ReplyChannel) -> - gen_server:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], []). + gen_server2:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], []). info(CPid) -> gen_server2:call(CPid, info). From b8084f29891f746dcfd66dc0cdcbcaa2e1c13022 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 22:24:23 +0800 Subject: [PATCH 27/41] Remove inc_stats/1 function --- src/emqttd_misc.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/emqttd_misc.erl b/src/emqttd_misc.erl index 69cba9d8f..e60d27d4f 100644 --- a/src/emqttd_misc.erl +++ b/src/emqttd_misc.erl @@ -19,7 +19,7 @@ -author("Feng Lee "). -export([merge_opts/2, start_timer/2, start_timer/3, cancel_timer/1, - proc_stats/0, proc_stats/1, inc_stats/1]). + proc_stats/0, proc_stats/1]). %% @doc Merge Options merge_opts(Defaults, Options) -> @@ -53,13 +53,13 @@ cancel_timer(Timer) -> _ -> ok end. +-spec(proc_stats() -> list()). proc_stats() -> proc_stats(self()). +-spec(proc_stats(pid()) -> list()). proc_stats(Pid) -> Stats = process_info(Pid, [message_queue_len, heap_size, reductions]), {value, {_, V}, Stats1} = lists:keytake(message_queue_len, 1, Stats), [{mailbox_len, V} | Stats1]. -inc_stats(Key) -> put(Key, get(Key) + 1). - From 239cf1b5fcb6efbffac7a67297ee2b7e2bdbb640 Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 23:07:27 +0800 Subject: [PATCH 28/41] Add '{backoff, 1000, 1000, 10000}' to return of init/1 --- src/emqttd_bridge.erl | 17 +++++++++-------- src/emqttd_pubsub.erl | 7 ++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/emqttd_bridge.erl b/src/emqttd_bridge.erl index 7e36ad256..7b3015f6d 100644 --- a/src/emqttd_bridge.erl +++ b/src/emqttd_bridge.erl @@ -40,16 +40,16 @@ qos = ?QOS_2, topic_suffix = <<>>, topic_prefix = <<>>, - mqueue :: emqttd_mqueue:mqueue(), + mqueue :: emqttd_mqueue:mqueue(), max_queue_len = 10000, ping_down_interval = ?PING_DOWN_INTERVAL, status = up}). --type(option() :: {qos, mqtt_qos()} | - {topic_suffix, binary()} | - {topic_prefix, binary()} | - {max_queue_len, pos_integer()} | - {ping_down_interval, pos_integer()}). +-type(option() :: {qos, mqtt_qos()} | + {topic_suffix, binary()} | + {topic_prefix, binary()} | + {max_queue_len, pos_integer()} | + {ping_down_interval, pos_integer()}). -export_type([option/0]). @@ -79,9 +79,10 @@ init([Pool, Id, Node, Topic, Options]) -> MQueue = emqttd_mqueue:new(qname(Node, Topic), [{max_len, State#state.max_queue_len}], emqttd_alarm:alarm_fun()), - {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}}; + {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}, hibernate, + {backoff, 1000, 1000, 10000}}; false -> - {stop, {cannot_connect, Node}} + {stop, {cannot_connect_node, Node}} end. parse_opts([], State) -> diff --git a/src/emqttd_pubsub.erl b/src/emqttd_pubsub.erl index c99aca513..393e4b788 100644 --- a/src/emqttd_pubsub.erl +++ b/src/emqttd_pubsub.erl @@ -164,11 +164,12 @@ pick(Subscriber) -> init([Pool, Id, Env]) -> ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, env = Env}}. + {ok, #state{pool = Pool, id = Id, env = Env}, hibernate, + {backoff, 2000, 2000, 20000}}. handle_call({subscribe, Topic, Subscriber, Options}, _From, State) -> add_subscriber(Topic, Subscriber, Options), - {reply, ok, setstats(State)}; + {reply, ok, setstats(State), hibernate}; handle_call({unsubscribe, Topic, Subscriber, Options}, _From, State) -> del_subscriber(Topic, Subscriber, Options), @@ -179,7 +180,7 @@ handle_call(Req, _From, State) -> handle_cast({subscribe, Topic, Subscriber, Options}, State) -> add_subscriber(Topic, Subscriber, Options), - {noreply, setstats(State)}; + {noreply, setstats(State), hibernate}; handle_cast({unsubscribe, Topic, Subscriber, Options}, State) -> del_subscriber(Topic, Subscriber, Options), From 365bfb9e80f79e03cf65239e509aa656c085432b Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 22 Feb 2017 23:08:22 +0800 Subject: [PATCH 29/41] Replace emqttd_misc:inc_stats/1 with inc_stats/1 --- src/emqttd_session.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 2a27f2906..06f988bb7 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -301,7 +301,7 @@ init([CleanSess, {ClientId, Username}, ClientPid]) -> created_at = os:timestamp()}, emqttd_sm:register_session(ClientId, CleanSess, info(State)), emqttd_hooks:run('session.created', [ClientId, Username]), - {ok, emit_stats(State), hibernate, {backoff, 1000, 1000, 10000}, ?MODULE}. + {ok, emit_stats(State), hibernate, {backoff, 1000, 1000, 10000}}. init_stats(Keys) -> lists:foreach(fun(K) -> put(K, 0) end, Keys). @@ -690,7 +690,7 @@ dispatch(Msg = #mqtt_message{qos = QoS}, end. enqueue_msg(Msg, State = #state{mqueue = Q}) -> - emqttd_misc:inc_stats(enqueue_msg), + inc_stats(enqueue_msg), State#state{mqueue = emqttd_mqueue:in(Msg, Q)}. %%-------------------------------------------------------------------- @@ -701,7 +701,7 @@ redeliver(Msg = #mqtt_message{qos = QoS}, State) -> deliver(Msg#mqtt_message{dup = if QoS =:= ?QOS2 -> false; true -> true end}, State). deliver(Msg, #state{client_pid = Pid}) -> - emqttd_misc:inc_stats(deliver_msg), + inc_stats(deliver_msg), Pid ! {deliver, Msg}. %%-------------------------------------------------------------------- @@ -793,6 +793,8 @@ emit_stats(State = #state{client_id = ClientId}) -> emqttd_stats:set_session_stats(ClientId, stats(State)), State. +inc_stats(Key) -> put(Key, get(Key) + 1). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 2d9dbe472948c0fe0b77f763901ad0e47c869fe7 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 11:55:52 +0800 Subject: [PATCH 30/41] Support to hook 'tag' function --- src/emqttd.erl | 21 ++++---- src/emqttd_hooks.erl | 114 +++++++++++++++++++++++++----------------- test/emqttd_SUITE.erl | 44 ++++++++-------- 3 files changed, 103 insertions(+), 76 deletions(-) diff --git a/src/emqttd.erl b/src/emqttd.erl index 63b53d4e1..037c0de1a 100644 --- a/src/emqttd.erl +++ b/src/emqttd.erl @@ -138,17 +138,20 @@ subscriber_down(Subscriber) -> %% Hooks API %%-------------------------------------------------------------------- --spec(hook(atom(), function(), list(any())) -> ok | {error, any()}). -hook(Hook, Function, InitArgs) -> - emqttd_hooks:add(Hook, Function, InitArgs). +-spec(hook(atom(), function() | {emqttd_hooks:hooktag(), function()}, list(any())) + -> ok | {error, any()}). +hook(Hook, TagFunction, InitArgs) -> + emqttd_hooks:add(Hook, TagFunction, InitArgs). --spec(hook(atom(), function(), list(any()), integer()) -> ok | {error, any()}). -hook(Hook, Function, InitArgs, Priority) -> - emqttd_hooks:add(Hook, Function, InitArgs, Priority). +-spec(hook(atom(), function() | {emqttd_hooks:hooktag(), function()}, list(any()), integer()) + -> ok | {error, any()}). +hook(Hook, TagFunction, InitArgs, Priority) -> + emqttd_hooks:add(Hook, TagFunction, InitArgs, Priority). --spec(unhook(atom(), function()) -> ok | {error, any()}). -unhook(Hook, Function) -> - emqttd_hooks:delete(Hook, Function). +-spec(unhook(atom(), function() | {emqttd_hooks:hooktag(), function()}) + -> ok | {error, any()}). +unhook(Hook, TagFunction) -> + emqttd_hooks:delete(Hook, TagFunction). -spec(run_hooks(atom(), list(any())) -> ok | stop). run_hooks(Hook, Args) -> diff --git a/src/emqttd_hooks.erl b/src/emqttd_hooks.erl index ce2691894..693a67ff7 100644 --- a/src/emqttd_hooks.erl +++ b/src/emqttd_hooks.erl @@ -32,7 +32,12 @@ -record(state, {}). --record(callback, {function :: function(), +-type(hooktag() :: atom() | string() | binary()). + +-export_type([hooktag/0]). + +-record(callback, {tag :: hooktag(), + function :: function(), init_args = [] :: list(any()), priority = 0 :: integer()}). @@ -47,17 +52,24 @@ start_link() -> %% Hooks API %%-------------------------------------------------------------------- --spec(add(atom(), function(), list(any())) -> ok). -add(HookPoint, Function, InitArgs) -> - add(HookPoint, Function, InitArgs, 0). +-spec(add(atom(), function() | {hooktag(), function()}, list(any())) -> ok). +add(HookPoint, Function, InitArgs) when is_function(Function) -> + add(HookPoint, {undefined, Function}, InitArgs, 0); --spec(add(atom(), function(), list(any()), integer()) -> ok). -add(HookPoint, Function, InitArgs, Priority) -> - gen_server:call(?MODULE, {add, HookPoint, Function, InitArgs, Priority}). +add(HookPoint, {Tag, Function}, InitArgs) when is_function(Function) -> + add(HookPoint, {Tag, Function}, InitArgs, 0). --spec(delete(atom(), function()) -> ok). -delete(HookPoint, Function) -> - gen_server:call(?MODULE, {delete, HookPoint, Function}). +-spec(add(atom(), function() | {hooktag(), function()}, list(any()), integer()) -> ok). +add(HookPoint, Function, InitArgs, Priority) when is_function(Function) -> + add(HookPoint, {undefined, Function}, InitArgs, Priority); +add(HookPoint, {Tag, Function}, InitArgs, Priority) when is_function(Function) -> + gen_server:call(?MODULE, {add, HookPoint, {Tag, Function}, InitArgs, Priority}). + +-spec(delete(atom(), function() | {hooktag(), function()}) -> ok). +delete(HookPoint, Function) when is_function(Function) -> + delete(HookPoint, {undefined, Function}); +delete(HookPoint, {Tag, Function}) when is_function(Function) -> + gen_server:call(?MODULE, {delete, HookPoint, {Tag, Function}}). %% @doc Run hooks without Acc. -spec(run(atom(), list(Arg :: any())) -> ok | stop). @@ -85,7 +97,8 @@ run_([#callback{function = Fun, init_args = InitArgs} | Callbacks], Args, Acc) - ok -> run_(Callbacks, Args, Acc); {ok, NewAcc} -> run_(Callbacks, Args, NewAcc); stop -> {stop, Acc}; - {stop, NewAcc} -> {stop, NewAcc} + {stop, NewAcc} -> {stop, NewAcc}; + _Any -> run_(Callbacks, Args, Acc) end; run_([], _Args, Acc) -> @@ -94,8 +107,8 @@ run_([], _Args, Acc) -> -spec(lookup(atom()) -> [#callback{}]). lookup(HookPoint) -> case ets:lookup(?HOOK_TAB, HookPoint) of - [] -> []; - [#hook{callbacks = Callbacks}] -> Callbacks + [#hook{callbacks = Callbacks}] -> Callbacks; + [] -> [] end. %%-------------------------------------------------------------------- @@ -106,39 +119,38 @@ init([]) -> ets:new(?HOOK_TAB, [set, protected, named_table, {keypos, #hook.name}]), {ok, #state{}}. -handle_call({add, HookPoint, Function, InitArgs, Priority}, _From, State) -> - Reply = - case ets:lookup(?HOOK_TAB, HookPoint) of - [#hook{callbacks = Callbacks}] -> - case lists:keyfind(Function, #callback.function, Callbacks) of - false -> - Callback = #callback{function = Function, - init_args = InitArgs, - priority = Priority}, - insert_hook_(HookPoint, add_callback_(Callback, Callbacks)); - _Callback -> - {error, already_hooked} - end; - [] -> - Callback = #callback{function = Function, - init_args = InitArgs, - priority = Priority}, - insert_hook_(HookPoint, [Callback]) - end, - {reply, Reply, State}; +handle_call({add, HookPoint, {Tag, Function}, InitArgs, Priority}, _From, State) -> + Callback = #callback{tag = Tag, function = Function, + init_args = InitArgs, priority = Priority}, + {reply, + case ets:lookup(?HOOK_TAB, HookPoint) of + [#hook{callbacks = Callbacks}] -> + case contain_(Tag, Function, Callbacks) of + false -> + insert_hook_(HookPoint, add_callback_(Callback, Callbacks)); + true -> + {error, already_hooked} + end; + [] -> + insert_hook_(HookPoint, [Callback]) + end, State}; -handle_call({delete, HookPoint, Function}, _From, State) -> - Reply = - case ets:lookup(?HOOK_TAB, HookPoint) of - [#hook{callbacks = Callbacks}] -> - insert_hook_(HookPoint, del_callback_(Function, Callbacks)); - [] -> - {error, not_found} - end, - {reply, Reply, State}; +handle_call({delete, HookPoint, {Tag, Function}}, _From, State) -> + {reply, + case ets:lookup(?HOOK_TAB, HookPoint) of + [#hook{callbacks = Callbacks}] -> + case contain_(Tag, Function, Callbacks) of + true -> + insert_hook_(HookPoint, del_callback_(Tag, Function, Callbacks)); + false -> + {error, not_found} + end; + [] -> + {error, not_found} + end, State}; -handle_call(_Req, _From, State) -> - {reply, ignore, State}. +handle_call(Req, _From, State) -> + {reply, {error, {unexpected_request, Req}}, State}. handle_cast(_Msg, State) -> {noreply, State}. @@ -162,6 +174,16 @@ insert_hook_(HookPoint, Callbacks) -> add_callback_(Callback, Callbacks) -> lists:keymerge(#callback.priority, Callbacks, [Callback]). -del_callback_(Function, Callbacks) -> - lists:keydelete(Function, #callback.function, Callbacks). +del_callback_(Tag, Function, Callbacks) -> + lists:filter( + fun(#callback{tag = Tag1, function = Func1}) -> + not ((Tag =:= Tag1) andalso (Function =:= Func1)) + end, Callbacks). + +contain_(_Tag, _Function, []) -> + false; +contain_(Tag, Function, [#callback{tag = Tag, function = Function}|_Callbacks]) -> + true; +contain_(Tag, Function, [_Callback | Callbacks]) -> + contain_(Tag, Function, Callbacks). diff --git a/test/emqttd_SUITE.erl b/test/emqttd_SUITE.erl index d7af619df..afa1a1f06 100644 --- a/test/emqttd_SUITE.erl +++ b/test/emqttd_SUITE.erl @@ -366,37 +366,39 @@ set_get_stat(_) -> %%-------------------------------------------------------------------- add_delete_hook(_) -> - emqttd:hook(test_hook, fun ?MODULE:hook_fun1/1, []), - emqttd:hook(test_hook, fun ?MODULE:hook_fun2/1, []), - {error, already_hooked} = emqttd:hook(test_hook, fun ?MODULE:hook_fun2/1, []), - Callbacks = [{callback, fun ?MODULE:hook_fun1/1, [], 0}, - {callback, fun ?MODULE:hook_fun2/1, [], 0}], + ok = emqttd:hook(test_hook, fun ?MODULE:hook_fun1/1, []), + ok = emqttd:hook(test_hook, {tag, fun ?MODULE:hook_fun2/1}, []), + {error, already_hooked} = emqttd:hook(test_hook, {tag, fun ?MODULE:hook_fun2/1}, []), + Callbacks = [{callback, undefined, fun ?MODULE:hook_fun1/1, [], 0}, + {callback, tag, fun ?MODULE:hook_fun2/1, [], 0}], Callbacks = emqttd_hooks:lookup(test_hook), - emqttd:unhook(test_hook, fun ?MODULE:hook_fun1/1), - emqttd:unhook(test_hook, fun ?MODULE:hook_fun2/1), - ok = emqttd:unhook(test_hook, fun ?MODULE:hook_fun2/1), - {error, not_found} = emqttd:unhook(test_hook1, fun ?MODULE:hook_fun2/1), + ok = emqttd:unhook(test_hook, fun ?MODULE:hook_fun1/1), + ct:print("Callbacks: ~p~n", [emqttd_hooks:lookup(test_hook)]), + ok = emqttd:unhook(test_hook, {tag, fun ?MODULE:hook_fun2/1}), + {error, not_found} = emqttd:unhook(test_hook1, {tag, fun ?MODULE:hook_fun2/1}), [] = emqttd_hooks:lookup(test_hook), - emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun1/1, [], 9), - emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun2/1, [], 8), - Callbacks2 = [{callback, fun ?MODULE:hook_fun2/1, [], 8}, - {callback, fun ?MODULE:hook_fun1/1, [], 9}], + ok = emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun1/1, [], 9), + ok = emqttd:hook(emqttd_hook, {"tag", fun ?MODULE:hook_fun2/1}, [], 8), + Callbacks2 = [{callback, "tag", fun ?MODULE:hook_fun2/1, [], 8}, + {callback, undefined, fun ?MODULE:hook_fun1/1, [], 9}], Callbacks2 = emqttd_hooks:lookup(emqttd_hook), - emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun1/1), - emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun2/1), + ok = emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun1/1), + ok = emqttd:unhook(emqttd_hook, {"tag", fun ?MODULE:hook_fun2/1}), [] = emqttd_hooks:lookup(emqttd_hook). run_hooks(_) -> - emqttd:hook(foldl_hook, fun ?MODULE:hook_fun3/4, [init]), - emqttd:hook(foldl_hook, fun ?MODULE:hook_fun4/4, [init]), - emqttd:hook(foldl_hook, fun ?MODULE:hook_fun5/4, [init]), + ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun3/4, [init]), + ok = emqttd:hook(foldl_hook, {tag, fun ?MODULE:hook_fun3/4}, [init]), + ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun4/4, [init]), + ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun5/4, [init]), {stop, [r3, r2]} = emqttd:run_hooks(foldl_hook, [arg1, arg2], []), {ok, []} = emqttd:run_hooks(unknown_hook, [], []), - emqttd:hook(foreach_hook, fun ?MODULE:hook_fun6/2, [initArg]), - emqttd:hook(foreach_hook, fun ?MODULE:hook_fun7/2, [initArg]), - emqttd:hook(foreach_hook, fun ?MODULE:hook_fun8/2, [initArg]), + ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun6/2, [initArg]), + ok = emqttd:hook(foreach_hook, {tag, fun ?MODULE:hook_fun6/2}, [initArg]), + ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun7/2, [initArg]), + ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun8/2, [initArg]), stop = emqttd:run_hooks(foreach_hook, [arg]). hook_fun1([]) -> ok. From ab76e7978b3d20adc844f49cb512b7e2114c920b Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 16:53:09 +0800 Subject: [PATCH 31/41] Add emqttd_gc module --- src/emqttd_gc.erl | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/emqttd_gc.erl diff --git a/src/emqttd_gc.erl b/src/emqttd_gc.erl new file mode 100644 index 000000000..339e895f3 --- /dev/null +++ b/src/emqttd_gc.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2013-2017 EMQ Enterprise, Inc. (http://emqtt.io) +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% GC Utility functions. + +-module(emqttd_gc). + +-author("Feng Lee "). + +-export([conn_max_gc_count/0, reset_conn_gc_count/2, maybe_force_gc/2]). + +-spec(conn_max_gc_count() -> integer()). +conn_max_gc_count() -> + case emqttd:env(conn_force_gc_count) of + undefined -> undefined; + I when I > 0 -> I + rand:uniform(I) + end. + +-spec(reset_conn_gc_count(pos_integer(), tuple()) -> tuple()). +reset_conn_gc_count(Pos, State) -> + case element(Pos, State) of + undefined -> State; + _I -> setelement(Pos, State, conn_max_gc_count()) + end. + +maybe_force_gc(Pos, State) -> + case element(Pos, State) of + undefined -> State; + I when I =< 0 -> garbage_collect(), + reset_conn_gc_count(Pos, State); + I -> setelement(Pos, State, I - 1) + end. + From 124aa454fbd5bfb6610bcfd51e4ea59aba6889d0 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 16:53:47 +0800 Subject: [PATCH 32/41] Add 'mqtt.conn.force_gc_count' config to tune GC of MQTT connection --- etc/emq.conf | 7 +++++++ priv/emq.schema | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/etc/emq.conf b/etc/emq.conf index f042bd279..4c3ab73c9 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -106,6 +106,13 @@ mqtt.max_clientid_len = 1024 ## Max Packet Size Allowed, 64K by default. mqtt.max_packet_size = 64KB +##-------------------------------------------------------------------- +## MQTT Connection +##-------------------------------------------------------------------- + +## Force GC: pos_integer +mqtt.conn.force_gc_count = 100 + ##-------------------------------------------------------------------- ## MQTT Client ##-------------------------------------------------------------------- diff --git a/priv/emq.schema b/priv/emq.schema index 1e3ba1778..92ca118d2 100644 --- a/priv/emq.schema +++ b/priv/emq.schema @@ -316,6 +316,15 @@ end}. {max_packet_size, cuttlefish:conf_get("mqtt.max_packet_size", Conf)}] end}. +%%-------------------------------------------------------------------- +%% MQTT Connection +%%-------------------------------------------------------------------- + +%% @doc Force the client to GC: integer +{mapping, "mqtt.conn.force_gc_count", "emqttd.conn_force_gc_count", [ + {datatype, integer} +]}. + %%-------------------------------------------------------------------- %% MQTT Client %%-------------------------------------------------------------------- From 7d65ad42add1ba667b10b5aa6dc7bfe22fdb5081 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 16:56:16 +0800 Subject: [PATCH 33/41] Add '[{fullsweep_after, 10}]' opts and 'force_gc_count' to tune the memory usage --- src/emqttd_client.erl | 32 ++++++++++++++++++-------------- src/emqttd_session.erl | 20 +++++++++++++++----- src/emqttd_ws_client.erl | 28 +++++++++++++++++----------- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index af36418d9..3cc7b341d 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -55,7 +55,7 @@ %% Unused fields: connname, peerhost, peerport -record(client_state, {connection, peername, conn_state, await_recv, rate_limit, packet_size, parser, proto_state, - keepalive, enable_stats}). + keepalive, enable_stats, force_gc_count}). -define(INFO_KEYS, [peername, conn_state, await_recv]). @@ -66,7 +66,7 @@ [esockd_net:format(State#client_state.peername) | Args])). start_link(Conn, Env) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]])}. + {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]], [{fullsweep_after, 10}])}. info(CPid) -> gen_server2:call(CPid, info). @@ -114,15 +114,17 @@ do_init(Conn, Env, Peername) -> Parser = emqttd_parser:initial_state(PacketSize), ProtoState = emqttd_protocol:init(Peername, SendFun, Env), EnableStats = get_value(client_enable_stats, Env, false), - State = run_socket(#client_state{connection = Conn, - peername = Peername, - await_recv = false, - conn_state = running, - rate_limit = RateLimit, - packet_size = PacketSize, - parser = Parser, - proto_state = ProtoState, - enable_stats = EnableStats}), + ForceGcCount = emqttd_gc:conn_max_gc_count(), + State = run_socket(#client_state{connection = Conn, + peername = Peername, + await_recv = false, + conn_state = running, + rate_limit = RateLimit, + packet_size = PacketSize, + parser = Parser, + proto_state = ProtoState, + enable_stats = EnableStats, + force_gc_count = ForceGcCount}), IdleTimout = get_value(client_idle_timeout, Env, 30000), gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, {backoff, 1000, 1000, 10000}). @@ -147,7 +149,7 @@ prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. handle_pre_hibernate(State) -> - {hibernate, emit_stats(State)}. + {hibernate, emit_stats(emqttd_gc:reset_conn_gc_count(State))}. handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> ProtoInfo = emqttd_protocol:info(ProtoState), @@ -237,7 +239,7 @@ handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> shutdown(Reason, State); handle_info({inet_reply, _Sock, ok}, State) -> - {noreply, State, hibernate}; + {noreply, gc(State), hibernate}; %% Tune GC handle_info({inet_reply, _Sock, {error, Reason}}, State) -> shutdown(Reason, State); @@ -291,7 +293,7 @@ code_change(_OldVsn, State, _Extra) -> %% Receive and parse tcp data received(<<>>, State) -> - {noreply, State, hibernate}; + {noreply, gc(State), hibernate}; received(Bytes, State = #client_state{parser = Parser, packet_size = PacketSize, @@ -370,3 +372,5 @@ shutdown(Reason, State) -> stop(Reason, State) -> {stop, Reason, State}. +gc(State) -> + emqttd_gc:maybe_force_gc(#client_state.force_gc_count, State). diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 06f988bb7..1c575b3ed 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -147,6 +147,9 @@ %% Enable Stats enable_stats :: boolean(), + %% Force GC Count + force_gc_count :: undefined | integer(), + created_at :: erlang:timestamp() }). @@ -157,7 +160,8 @@ -define(STATE_KEYS, [clean_sess, client_id, username, binding, client_pid, old_client_pid, next_msg_id, max_subscriptions, subscriptions, upgrade_qos, inflight, max_inflight, retry_interval, mqueue, awaiting_rel, max_awaiting_rel, - await_rel_timeout, expiry_interval, enable_stats, created_at]). + await_rel_timeout, expiry_interval, enable_stats, force_gc_count, + created_at]). -define(LOG(Level, Format, Args, State), lager:Level([{client, State#state.client_id}], @@ -166,7 +170,8 @@ %% @doc Start a Session -spec(start_link(boolean(), {mqtt_client_id(), mqtt_username()}, pid()) -> {ok, pid()} | {error, any()}). start_link(CleanSess, {ClientId, Username}, ClientPid) -> - gen_server2:start_link(?MODULE, [CleanSess, {ClientId, Username}, ClientPid], []). + gen_server2:start_link(?MODULE, [CleanSess, {ClientId, Username}, ClientPid], + [{fullsweep_after, 10}]). %% Tune GC. %%-------------------------------------------------------------------- %% PubSub API @@ -280,6 +285,7 @@ init([CleanSess, {ClientId, Username}, ClientPid]) -> {ok, QEnv} = emqttd:env(queue), MaxInflight = get_value(max_inflight, Env, 0), EnableStats = get_value(enable_stats, Env, false), + ForceGcCount = emqttd_gc:conn_max_gc_count(), MQueue = emqttd_mqueue:new(ClientId, QEnv, emqttd_alarm:alarm_fun()), State = #state{clean_sess = CleanSess, binding = binding(ClientPid), @@ -298,6 +304,7 @@ init([CleanSess, {ClientId, Username}, ClientPid]) -> max_awaiting_rel = get_value(max_awaiting_rel, Env), expiry_interval = get_value(expiry_interval, Env), enable_stats = EnableStats, + force_gc_count = ForceGcCount, created_at = os:timestamp()}, emqttd_sm:register_session(ClientId, CleanSess, info(State)), emqttd_hooks:run('session.created', [ClientId, Username]), @@ -334,7 +341,7 @@ prioritise_info(Msg, _Len, _State) -> end. handle_pre_hibernate(State) -> - {hibernate, emit_stats(State)}. + {hibernate, emit_stats(emqttd_gc:reset_conn_gc_count(State))}. handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, State = #state{awaiting_rel = AwaitingRel, @@ -443,7 +450,7 @@ handle_cast({pubrel, PacketId}, State = #state{awaiting_rel = AwaitingRel}) -> case maps:take(PacketId, AwaitingRel) of {Msg, AwaitingRel1} -> spawn(emqttd_server, publish, [Msg]), %%:) - State#state{awaiting_rel = AwaitingRel1}; + gc(State#state{awaiting_rel = AwaitingRel1}); error -> ?LOG(warning, "Cannot find PUBREL: ~p", [PacketId], State), emqttd_metrics:inc('packets/pubrel/missed'), @@ -521,7 +528,7 @@ handle_cast(Msg, State) -> %% Dispatch Message handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, mqtt_message) -> - {noreply, dispatch(tune_qos(Topic, Msg, State), State), hibernate}; + {noreply, gc(dispatch(tune_qos(Topic, Msg, State), State)), hibernate}; %% Do nothing if the client has been disconnected. handle_info({timeout, _Timer, retry_delivery}, State = #state{client_pid = undefined}) -> @@ -808,3 +815,6 @@ hibernate(State) -> shutdown(Reason, State) -> {stop, {shutdown, Reason}, State}. +gc(State) -> + emqttd_gc:maybe_force_gc(#state.force_gc_count, State). + diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index b4c9d4fe9..4faa44624 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -49,7 +49,7 @@ %% WebSocket Client State -record(wsclient_state, {ws_pid, peername, connection, proto_state, keepalive, - enable_stats}). + enable_stats, force_gc_count}). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -59,7 +59,8 @@ %% @doc Start WebSocket Client. start_link(Env, WsPid, Req, ReplyChannel) -> - gen_server2:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], []). + gen_server2:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], + [{fullsweep_after, 10}]). %% Tune GC. info(CPid) -> gen_server2:call(CPid, info). @@ -93,11 +94,13 @@ init([Env, WsPid, Req, ReplyChannel]) -> [{ws_initial_headers, Headers} | Env]), IdleTimeout = get_value(client_idle_timeout, Env, 30000), EnableStats = get_value(client_enable_stats, Env, false), - {ok, #wsclient_state{ws_pid = WsPid, - peername = Peername, - connection = Req:get(connection), - proto_state = ProtoState, - enable_stats = EnableStats}, + ForceGcCount = emqttd_gc:conn_max_gc_count(), + {ok, #wsclient_state{ws_pid = WsPid, + peername = Peername, + connection = Req:get(connection), + proto_state = ProtoState, + enable_stats = EnableStats, + force_gc_count = ForceGcCount}, IdleTimeout, {backoff, 1000, 1000, 10000}, ?MODULE}. prioritise_call(Msg, _From, _Len, _State) -> @@ -108,7 +111,7 @@ prioritise_info(Msg, _Len, _State) -> handle_pre_hibernate(State = #wsclient_state{ws_pid = WsPid}) -> erlang:garbage_collect(WsPid),%%TODO: [{async, RequestId}]?? - {hibernate, emit_stats(State)}. + {hibernate, emqttd_gc:reset_conn_gc_count(emit_stats(State))}. handle_call(info, From, State = #wsclient_state{peername = Peername, proto_state = ProtoState}) -> @@ -135,7 +138,7 @@ handle_cast({received, Packet}, State = #wsclient_state{proto_state = ProtoState emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - {noreply, State#wsclient_state{proto_state = ProtoState1}, hibernate}; + {noreply, gc(State#wsclient_state{proto_state = ProtoState1}), hibernate}; {error, Error} -> ?WSLOG(error, "Protocol error - ~p", [Error], State), shutdown(Error, State); @@ -172,7 +175,7 @@ handle_info({deliver, Message}, State) -> with_proto( fun(ProtoState) -> emqttd_protocol:send(Message, ProtoState) - end, State); + end, gc(State)); handle_info({redeliver, {?PUBREL, PacketId}}, State) -> with_proto( @@ -277,6 +280,9 @@ reply(Reply, State) -> shutdown(Reason, State) -> stop({shutdown, Reason}, State). -stop(Reason, State ) -> +stop(Reason, State) -> {stop, Reason, State}. +gc(State) -> + emqttd_gc:maybe_force_gc(#wsclient_state.force_gc_count, State). + From 73847b96fcd0eed8f6317fe7ca481cd7b50f634d Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 17:16:55 +0800 Subject: [PATCH 34/41] Add 'FULLSWEEP_OPTS' macro --- include/emqttd_internal.hrl | 2 ++ src/emqttd_client.erl | 3 ++- src/emqttd_session.erl | 2 +- src/emqttd_ws_client.erl | 4 +++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/emqttd_internal.hrl b/include/emqttd_internal.hrl index ec5fb3e73..fc482313f 100644 --- a/include/emqttd_internal.hrl +++ b/include/emqttd_internal.hrl @@ -58,3 +58,5 @@ false-> (FalseFun) end)). +-define(FULLSWEEP_OPTS, [{fullsweep_after, 10}]). + diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 3cc7b341d..cd7ab45d0 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -66,7 +66,8 @@ [esockd_net:format(State#client_state.peername) | Args])). start_link(Conn, Env) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]], [{fullsweep_after, 10}])}. + {ok, proc_lib:spawn_opt(?MODULE, init, [[Conn, Env]], + [{spawn_opt, [link | ?FULLSWEEP_OPTS]}])}. info(CPid) -> gen_server2:call(CPid, info). diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 1c575b3ed..1ee57b575 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -171,7 +171,7 @@ -spec(start_link(boolean(), {mqtt_client_id(), mqtt_username()}, pid()) -> {ok, pid()} | {error, any()}). start_link(CleanSess, {ClientId, Username}, ClientPid) -> gen_server2:start_link(?MODULE, [CleanSess, {ClientId, Username}, ClientPid], - [{fullsweep_after, 10}]). %% Tune GC. + [{spawn_opt, ?FULLSWEEP_OPTS}]). %% Tune GC. %%-------------------------------------------------------------------- %% PubSub API diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 4faa44624..1d181e1bb 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -26,6 +26,8 @@ -include("emqttd_protocol.hrl"). +-include("emqttd_internal.hrl"). + -import(proplists, [get_value/3]). %% API Exports @@ -60,7 +62,7 @@ %% @doc Start WebSocket Client. start_link(Env, WsPid, Req, ReplyChannel) -> gen_server2:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], - [{fullsweep_after, 10}]). %% Tune GC. + [{spawn_opt, ?FULLSWEEP_OPTS}]). %% Tune GC. info(CPid) -> gen_server2:call(CPid, info). From e972103f74f494419d0801b0a08a0b2185d8e807 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 17:25:44 +0800 Subject: [PATCH 35/41] Fix the 'spawn_opt' options --- src/emqttd_client.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index cd7ab45d0..f7571efb9 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -66,8 +66,7 @@ [esockd_net:format(State#client_state.peername) | Args])). start_link(Conn, Env) -> - {ok, proc_lib:spawn_opt(?MODULE, init, [[Conn, Env]], - [{spawn_opt, [link | ?FULLSWEEP_OPTS]}])}. + {ok, proc_lib:spawn_opt(?MODULE, init, [[Conn, Env]], [link | ?FULLSWEEP_OPTS])}. info(CPid) -> gen_server2:call(CPid, info). From 5ef4fce141d46f1f1937d979a74ffff0e2ec04da Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 17:26:29 +0800 Subject: [PATCH 36/41] Handle the {ok, I} return --- src/emqttd_gc.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqttd_gc.erl b/src/emqttd_gc.erl index 339e895f3..9261e297e 100644 --- a/src/emqttd_gc.erl +++ b/src/emqttd_gc.erl @@ -25,8 +25,8 @@ -spec(conn_max_gc_count() -> integer()). conn_max_gc_count() -> case emqttd:env(conn_force_gc_count) of - undefined -> undefined; - I when I > 0 -> I + rand:uniform(I) + {ok, I} when I > 0 -> I + rand:uniform(I); + undefined -> undefined end. -spec(reset_conn_gc_count(pos_integer(), tuple()) -> tuple()). From 1e3675028873e744a3f07dc608e922f5c839603f Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 17:40:50 +0800 Subject: [PATCH 37/41] Use emqttd_gc:reset_conn_gc_count/2 API --- src/emqttd_client.erl | 2 +- src/emqttd_session.erl | 2 +- src/emqttd_ws_client.erl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index f7571efb9..e3e68b8e4 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -149,7 +149,7 @@ prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. handle_pre_hibernate(State) -> - {hibernate, emit_stats(emqttd_gc:reset_conn_gc_count(State))}. + {hibernate, emqttd_gc:reset_conn_gc_count(#client_state.force_gc_count, emit_stats(State))}. handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> ProtoInfo = emqttd_protocol:info(ProtoState), diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 1ee57b575..288565a3a 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -341,7 +341,7 @@ prioritise_info(Msg, _Len, _State) -> end. handle_pre_hibernate(State) -> - {hibernate, emit_stats(emqttd_gc:reset_conn_gc_count(State))}. + {hibernate, emqttd_gc:reset_conn_gc_count(#state.force_gc_count, emit_stats(State))}. handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, State = #state{awaiting_rel = AwaitingRel, diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 1d181e1bb..bde322dcf 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -112,8 +112,8 @@ prioritise_info(Msg, _Len, _State) -> case Msg of {redeliver, _} -> 5; _ -> 0 end. handle_pre_hibernate(State = #wsclient_state{ws_pid = WsPid}) -> - erlang:garbage_collect(WsPid),%%TODO: [{async, RequestId}]?? - {hibernate, emqttd_gc:reset_conn_gc_count(emit_stats(State))}. + erlang:garbage_collect(WsPid), + {hibernate, emqttd_gc:reset_conn_gc_count(#wsclient_state.force_gc_count, emit_stats(State))}. handle_call(info, From, State = #wsclient_state{peername = Peername, proto_state = ProtoState}) -> From edd99dc5ed3e9b9408d3e30670ab6c252c0cef80 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 23 Feb 2017 18:53:16 +0800 Subject: [PATCH 38/41] Disable the force GC if conn_force_gc_count = 0 --- etc/emq.conf | 2 +- src/emqttd_gc.erl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/emq.conf b/etc/emq.conf index 4c3ab73c9..5f43eff51 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -110,7 +110,7 @@ mqtt.max_packet_size = 64KB ## MQTT Connection ##-------------------------------------------------------------------- -## Force GC: pos_integer +## Force GC: integer. Value 0 disabled the Force GC. mqtt.conn.force_gc_count = 100 ##-------------------------------------------------------------------- diff --git a/src/emqttd_gc.erl b/src/emqttd_gc.erl index 9261e297e..04cdbf2d5 100644 --- a/src/emqttd_gc.erl +++ b/src/emqttd_gc.erl @@ -26,6 +26,7 @@ conn_max_gc_count() -> case emqttd:env(conn_force_gc_count) of {ok, I} when I > 0 -> I + rand:uniform(I); + {ok, I} when I =< 0 -> undefined; undefined -> undefined end. From 9309baf17cade72873b65ac1bc5eb22a9be467c9 Mon Sep 17 00:00:00 2001 From: Feng Date: Thu, 23 Feb 2017 22:59:26 +0800 Subject: [PATCH 39/41] Rename mqtt_msgid() type to mqtt_msg_id() --- include/emqttd.hrl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/emqttd.hrl b/include/emqttd.hrl index 3f4f22229..a65086858 100644 --- a/include/emqttd.hrl +++ b/include/emqttd.hrl @@ -58,6 +58,7 @@ %%-------------------------------------------------------------------- %% MQTT Subscription %%-------------------------------------------------------------------- + -record(mqtt_subscription, { subid :: binary() | atom(), topic :: binary(), @@ -104,7 +105,7 @@ %% MQTT Message %%-------------------------------------------------------------------- --type(mqtt_msgid() :: binary() | undefined). +-type(mqtt_msg_id() :: binary() | undefined). -type(mqtt_pktid() :: 1..16#ffff | undefined). @@ -112,7 +113,7 @@ -record(mqtt_message, { %% Global unique message ID - id :: mqtt_msgid(), + id :: mqtt_msg_id(), %% PacketId pktid :: mqtt_pktid(), %% ClientId and Username From 440011da9a6c503e074c676190bea1866b0951f4 Mon Sep 17 00:00:00 2001 From: Feng Date: Thu, 23 Feb 2017 23:00:55 +0800 Subject: [PATCH 40/41] Rename 'Pkt' to 'PktCnt', 'Msg' to 'MsgCnt' --- src/emqttd_protocol.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index 4b3c3ca76..b87274455 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -334,12 +334,12 @@ inc_stats(_Direct, _Type, Stats = #proto_stats{enable_stats = false}) -> Stats; inc_stats(recv, Type, Stats) -> - #proto_stats{recv_pkt = Pkt, recv_msg = Msg} = Stats, - inc_stats(Type, #proto_stats.recv_pkt, Pkt, #proto_stats.recv_msg, Msg, Stats); + #proto_stats{recv_pkt = PktCnt, recv_msg = MsgCnt} = Stats, + inc_stats(Type, #proto_stats.recv_pkt, PktCnt, #proto_stats.recv_msg, MsgCnt, Stats); inc_stats(send, Type, Stats) -> - #proto_stats{send_pkt = Pkt, send_msg = Msg} = Stats, - inc_stats(Type, #proto_stats.send_pkt, Pkt, #proto_stats.send_msg, Msg, Stats). + #proto_stats{send_pkt = PktCnt, send_msg = MsgCnt} = Stats, + inc_stats(Type, #proto_stats.send_pkt, PktCnt, #proto_stats.send_msg, MsgCnt, Stats). inc_stats(Type, PktPos, PktCnt, MsgPos, MsgCnt, Stats) -> Stats1 = setelement(PktPos, Stats, PktCnt + 1), From 1e91c0e220b4ee84e4dc98a152963f24fe8cc882 Mon Sep 17 00:00:00 2001 From: Feng Date: Thu, 23 Feb 2017 23:01:20 +0800 Subject: [PATCH 41/41] Format code --- src/emqttd_bridge.erl | 4 ++-- src/emqttd_pubsub.erl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqttd_bridge.erl b/src/emqttd_bridge.erl index 7b3015f6d..a1dd34bcc 100644 --- a/src/emqttd_bridge.erl +++ b/src/emqttd_bridge.erl @@ -79,8 +79,8 @@ init([Pool, Id, Node, Topic, Options]) -> MQueue = emqttd_mqueue:new(qname(Node, Topic), [{max_len, State#state.max_queue_len}], emqttd_alarm:alarm_fun()), - {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}, hibernate, - {backoff, 1000, 1000, 10000}}; + {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}, + hibernate, {backoff, 1000, 1000, 10000}}; false -> {stop, {cannot_connect_node, Node}} end. diff --git a/src/emqttd_pubsub.erl b/src/emqttd_pubsub.erl index 393e4b788..d976618cd 100644 --- a/src/emqttd_pubsub.erl +++ b/src/emqttd_pubsub.erl @@ -164,8 +164,8 @@ pick(Subscriber) -> init([Pool, Id, Env]) -> ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, env = Env}, hibernate, - {backoff, 2000, 2000, 20000}}. + {ok, #state{pool = Pool, id = Id, env = Env}, + hibernate, {backoff, 2000, 2000, 20000}}. handle_call({subscribe, Topic, Subscriber, Options}, _From, State) -> add_subscriber(Topic, Subscriber, Options),