From 698d096caeb396ed1a89de5328c569125bd44c99 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 23 Oct 2015 17:54:31 +0800 Subject: [PATCH 01/37] +zdbbl --- rel/files/vm.args | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rel/files/vm.args b/rel/files/vm.args index ef3307fed..8e37830dc 100644 --- a/rel/files/vm.args +++ b/rel/files/vm.args @@ -28,6 +28,10 @@ ## max atom number ## +t +## Set the distribution buffer busy limit (dist_buf_busy_limit) in kilobytes. +## Valid range is 1-2097151. Default is 1024. +## +zdbbl 8192 + ##------------------------------------------------------------------------- ## Env ##------------------------------------------------------------------------- From 676aa3cb7128d54bacf21f5b4f49bff9d0602df1 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 23 Oct 2015 18:05:08 +0800 Subject: [PATCH 02/37] order of sockopts --- rel/files/emqttd.config.development | 4 ++-- rel/files/emqttd.config.production | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rel/files/emqttd.config.development b/rel/files/emqttd.config.development index 4989110f9..fff25aed9 100644 --- a/rel/files/emqttd.config.development +++ b/rel/files/emqttd.config.development @@ -187,11 +187,11 @@ {access, [{allow, all}]}, %% Socket Options {sockopts, [ - {backlog, 512} %Set buffer if hight thoughtput %{recbuf, 4096}, - %{sndbuf, 4096} + %{sndbuf, 4096}, %{buffer, 4096}, + {backlog, 512} ]} ]}, {mqtts, 8883, [ diff --git a/rel/files/emqttd.config.production b/rel/files/emqttd.config.production index 53b1ea3a0..2f70b61f3 100644 --- a/rel/files/emqttd.config.production +++ b/rel/files/emqttd.config.production @@ -179,11 +179,11 @@ {access, [{allow, all}]}, %% Socket Options {sockopts, [ - {backlog, 512} %Set buffer if hight thoughtput %{recbuf, 4096}, - %{sndbuf, 4096} + %{sndbuf, 4096}, %{buffer, 4096}, + {backlog, 512} ]} ]}, {mqtts, 8883, [ From 59ca283eb020a05d10e2422cde4bf798663bfc84 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 23 Oct 2015 18:06:35 +0800 Subject: [PATCH 03/37] gen_server --- src/emqttd_ws_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 3d06e8432..4d72ce7bb 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -115,7 +115,7 @@ reset_parser(State = #wsocket_state{packet_opts = PktOpts}) -> State#wsocket_state{parser = emqttd_parser:new(PktOpts)}. %%%============================================================================= -%%% gen_fsm callbacks +%%% gen_server callbacks %%%============================================================================= init([WsPid, Req, ReplyChannel, PktOpts]) -> From 09047287c4230efe7e85db647e188f339c141cb6 Mon Sep 17 00:00:00 2001 From: Feng Date: Sun, 25 Oct 2015 14:56:26 +0800 Subject: [PATCH 04/37] retained test --- test/emqttd_retained_tests.erl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/emqttd_retained_tests.erl diff --git a/test/emqttd_retained_tests.erl b/test/emqttd_retained_tests.erl new file mode 100644 index 000000000..b541a0731 --- /dev/null +++ b/test/emqttd_retained_tests.erl @@ -0,0 +1,14 @@ +-module(emqttd_retained_tests). + +-include("emqttd.hrl"). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +retain_test() -> + mnesia:start(), + emqttd_retained:mnesia(boot), + mnesia:stop(). + +-endif. From 9a590b2f39f2bd33c97b9a669fe645d20c789d93 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:19:32 +0800 Subject: [PATCH 05/37] rate limit --- rel/files/emqttd.config.development | 2 ++ rel/files/emqttd.config.production | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rel/files/emqttd.config.development b/rel/files/emqttd.config.development index fff25aed9..6ec4754c6 100644 --- a/rel/files/emqttd.config.development +++ b/rel/files/emqttd.config.development @@ -185,6 +185,8 @@ {max_clients, 512}, %% Socket Access Control {access, [{allow, all}]}, + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec + %% {rate_limit, "100,10"}, %% 100K burst, 10K rate %% Socket Options {sockopts, [ %Set buffer if hight thoughtput diff --git a/rel/files/emqttd.config.production b/rel/files/emqttd.config.production index 2f70b61f3..e3685518f 100644 --- a/rel/files/emqttd.config.production +++ b/rel/files/emqttd.config.production @@ -175,6 +175,8 @@ {acceptors, 16}, %% Maximum number of concurrent clients {max_clients, 8192}, + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec. + %% {rate_limit, "10,1"}, %% 10K burst, 1K rate %% Socket Access Control {access, [{allow, all}]}, %% Socket Options From fb9f1bf8e4fcbe8d0954008f352f797011d60eec Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:20:13 +0800 Subject: [PATCH 06/37] rate_limit --- src/emqttd.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/emqttd.erl b/src/emqttd.erl index eeb5bab4d..09661d6c8 100644 --- a/src/emqttd.erl +++ b/src/emqttd.erl @@ -91,7 +91,8 @@ open_listener({https, Port, Options}) -> mochiweb:start_http(Port, Options, MFArgs). open_listener(Protocol, Port, Options) -> - MFArgs = {emqttd_client, start_link, [env(mqtt)]}, + Rl = rate_limiter(emqttd_opts:g(rate_limit, Options)), + MFArgs = {emqttd_client, start_link, [[{rate_limiter, Rl} | env(mqtt)]]}, esockd:open(Protocol, Port, merge_sockopts(Options) , MFArgs). merge_sockopts(Options) -> @@ -99,6 +100,14 @@ merge_sockopts(Options) -> proplists:get_value(sockopts, Options, [])), emqttd_opts:merge(Options, [{sockopts, SockOpts}]). +%% TODO: will refactor in 0.14.0 release. +rate_limiter(undefined) -> + undefined; +rate_limiter(Config) -> + Bps = fun(S) -> list_to_integer(string:strip(S)) * 1024 end, + [Burst, Rate] = [Bps(S) || S <- string:tokens(Config, ",")], + esockd_rate_limiter:new(Burst, Rate). + %%------------------------------------------------------------------------------ %% @doc Close Listeners %% @end From 2af91ea1401574f1de099ffe0d7c35f4c5da8b77 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:21:03 +0800 Subject: [PATCH 07/37] link with client --- src/emqttd_session.erl | 44 ++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index d2163d3f4..f33e72521 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -81,9 +81,6 @@ %% Client Pid bind with session client_pid :: pid(), - %% Client Monitor - client_mon :: reference(), - %% Last packet id of the session packet_id = 1, @@ -224,7 +221,8 @@ unsubscribe(SessPid, Topics) -> %%%============================================================================= init([CleanSess, ClientId, ClientPid]) -> - %% process_flag(trap_exit, true), + process_flag(trap_exit, true), + true = link(ClientPid), QEnv = emqttd:env(mqtt, queue), SessEnv = emqttd:env(mqtt, session), Session = #session{ @@ -245,10 +243,8 @@ init([CleanSess, ClientId, ClientPid]) -> collect_interval = emqttd_opts:g(collect_interval, SessEnv, 0), timestamp = os:timestamp()}, emqttd_sm:register_session(CleanSess, ClientId, info(Session)), - %% monitor client - MRef = erlang:monitor(process, ClientPid), %% start statistics - {ok, start_collector(Session#session{client_mon = MRef}), hibernate}. + {ok, start_collector(Session), hibernate}. prioritise_call(Msg, _From, _Len, _State) -> case Msg of _ -> 0 end. @@ -268,7 +264,6 @@ prioritise_cast(Msg, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of - {'DOWN', _, _, _, _} -> 10; {'EXIT', _, _} -> 10; session_expired -> 10; {timeout, _, _} -> 5; @@ -368,7 +363,6 @@ handle_cast({resume, ClientId, ClientPid}, Session) -> #session{client_id = ClientId, client_pid = OldClientPid, - client_mon = MRef, inflight_queue = InflightQ, awaiting_ack = AwaitingAck, awaiting_comp = AwaitingComp, @@ -388,10 +382,12 @@ handle_cast({resume, ClientId, ClientPid}, Session) -> true -> lager:error([{client, ClientId}], "Session(~s): ~p kickout ~p", [ClientId, ClientPid, OldClientPid]), - OldClientPid ! {stop, duplicate_id, ClientPid}, - erlang:demonitor(MRef, [flush]) + unlink(OldClientPid), + OldClientPid ! {stop, duplicate_id, ClientPid} end, + true = link(ClientPid), + %% Redeliver PUBREL [ClientPid ! {redeliver, {?PUBREL, PktId}} || PktId <- maps:keys(AwaitingComp)], @@ -402,7 +398,6 @@ handle_cast({resume, ClientId, ClientPid}, Session) -> [cancel_timer(TRef) || TRef <- maps:values(AwaitingComp)], Session1 = Session#session{client_pid = ClientPid, - client_mon = erlang:monitor(process, ClientPid), awaiting_ack = #{}, awaiting_comp = #{}, expired_timer = undefined}, @@ -548,21 +543,24 @@ handle_info(collect_info, Session = #session{clean_sess = CleanSess, client_id = emqttd_sm:register_session(CleanSess, ClientId, info(Session)), {noreply, start_collector(Session), hibernate}; -handle_info({'DOWN', _MRef, process, ClientPid, _}, Session = #session{clean_sess = true, - client_pid = ClientPid}) -> +handle_info({'EXIT', ClientPid, _Reason}, Session = #session{clean_sess = true, + client_pid = ClientPid}) -> {stop, normal, Session}; -handle_info({'DOWN', _MRef, process, ClientPid, _}, Session = #session{clean_sess = false, - client_pid = ClientPid, - expired_after = Expires}) -> +handle_info({'EXIT', ClientPid, Reason}, Session = #session{clean_sess = false, + client_id = ClientId, + client_pid = ClientPid, + expired_after = Expires}) -> + lager:info("Session(~s): unlink with client ~p: reason=~p", + [ClientId, ClientPid, Reason]), TRef = timer(Expires, session_expired), - noreply(Session#session{client_pid = undefined, client_mon = undefined, expired_timer = TRef}); + noreply(Session#session{client_pid = undefined, expired_timer = TRef}); -handle_info({'DOWN', _MRef, process, Pid, Reason}, Session = #session{client_id = ClientId, - client_pid = ClientPid}) -> - lager:error([{client, ClientId}], "Session(~s): unexpected DOWN: " - "client_pid=~p, down_pid=~p, reason=~p", - [ClientId, ClientPid, Pid, Reason]), +handle_info({'EXIT', Pid, Reason}, Session = #session{client_id = ClientId, + client_pid = ClientPid}) -> + + lager:error("Session(~s): Unexpected EXIT: client_pid=~p, exit_pid=~p, reason=~p", + [ClientId, ClientPid, Pid, Reason]), noreply(Session); handle_info(session_expired, Session = #session{client_id = ClientId}) -> From f58f42196abdc61edc67e38c7a223873fba51d9f Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:21:24 +0800 Subject: [PATCH 08/37] 0.12.4 --- src/emqttd.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd.app.src b/src/emqttd.app.src index c35f7465d..7591c61d5 100644 --- a/src/emqttd.app.src +++ b/src/emqttd.app.src @@ -1,7 +1,7 @@ {application, emqttd, [ {id, "emqttd"}, - {vsn, "0.12.3"}, + {vsn, "0.12.4"}, {description, "Erlang MQTT Broker"}, {modules, []}, {registered, []}, From 976c7653f3ed30ff50b0151f6e78a0130c53ae86 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:25:40 +0800 Subject: [PATCH 09/37] use esockd_rate_limiter --- src/emqttd_throttle.erl | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 src/emqttd_throttle.erl diff --git a/src/emqttd_throttle.erl b/src/emqttd_throttle.erl deleted file mode 100644 index 256ae27d4..000000000 --- a/src/emqttd_throttle.erl +++ /dev/null @@ -1,32 +0,0 @@ -%%%----------------------------------------------------------------------------- -%%% Copyright (c) 2012-2015 eMQTT.IO, All Rights Reserved. -%%% -%%% Permission is hereby granted, free of charge, to any person obtaining a copy -%%% of this software and associated documentation files (the "Software"), to deal -%%% in the Software without restriction, including without limitation the rights -%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -%%% copies of the Software, and to permit persons to whom the Software is -%%% furnished to do so, subject to the following conditions: -%%% -%%% The above copyright notice and this permission notice shall be included in all -%%% copies or substantial portions of the Software. -%%% -%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -%%% SOFTWARE. -%%%----------------------------------------------------------------------------- -%%% @doc -%%% emqttd client throttle. -%%% -%%% @end -%%%----------------------------------------------------------------------------- --module(emqttd_throttle). - --author("Feng Lee "). - -%% TODO:... 0.11.0... - From 47710c36aa406b95171d593f98b11fbc83adfa38 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 26 Oct 2015 09:25:57 +0800 Subject: [PATCH 10/37] port_command --- src/emqttd_client.erl | 126 +++++++++++++++++++++++++----------------- 1 file changed, 75 insertions(+), 51 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index de7c10766..b8909ff32 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -20,7 +20,7 @@ %%% SOFTWARE. %%%----------------------------------------------------------------------------- %%% @doc -%%% MQTT Client +%%% MQTT Client Connection. %%% %%% @end %%%----------------------------------------------------------------------------- @@ -52,7 +52,7 @@ conn_name, await_recv, conn_state, - conserve, + rate_limiter, parser, proto_state, packet_opts, @@ -85,22 +85,26 @@ unsubscribe(CPid, Topics) -> init([SockArgs = {Transport, Sock, _SockFun}, MqttEnv]) -> % Transform if ssl. - {ok, NewSock} = esockd_connection:accept(SockArgs), + {ok, NewSock} = esockd_connection:accept(SockArgs), + %%TODO:... + {ok, BufSizes} = inet:getopts(Sock, [sndbuf, recbuf, buffer]), + io:format("~p~n", [BufSizes]), {ok, Peername} = emqttd_net:peername(Sock), - {ok, ConnStr} = emqttd_net:connection_string(Sock, inbound), - SendFun = fun(Data) -> Transport:send(NewSock, Data) end, - PktOpts = proplists:get_value(packet, MqttEnv), + {ok, ConnStr} = emqttd_net:connection_string(Sock, inbound), + SendFun = send_fun(Transport, NewSock), + PktOpts = proplists:get_value(packet, MqttEnv), ProtoState = emqttd_protocol:init(Peername, SendFun, PktOpts), - State = control_throttle(#state{transport = Transport, - socket = NewSock, - peername = Peername, - conn_name = ConnStr, - await_recv = false, - conn_state = running, - conserve = false, - packet_opts = PktOpts, - parser = emqttd_parser:new(PktOpts), - proto_state = ProtoState}), + Limiter = proplists:get_value(rate_limiter, MqttEnv), + State = run_socket(#state{transport = Transport, + socket = NewSock, + peername = Peername, + conn_name = ConnStr, + await_recv = false, + conn_state = running, + rate_limiter = Limiter, + packet_opts = PktOpts, + parser = emqttd_parser:new(PktOpts), + proto_state = ProtoState}), ClientOpts = proplists:get_value(client, MqttEnv), IdleTimout = proplists:get_value(idle_timeout, ClientOpts, 10), gen_server:enter_loop(?MODULE, [], State, timer:seconds(IdleTimout)). @@ -146,20 +150,26 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State = #state{proto_state = Prot {ok, ProtoState1} = emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState), noreply(State#state{proto_state = ProtoState1}); -handle_info({inet_reply, _Ref, ok}, State) -> - noreply(State); +handle_info(activate_sock, State) -> + noreply(run_socket(State#state{conn_state = running})); handle_info({inet_async, Sock, _Ref, {ok, Data}}, State = #state{peername = Peername, socket = Sock}) -> + Size = size(Data), lager:debug("RECV from ~s: ~p", [emqttd_net:format(Peername), Data]), - emqttd_metrics:inc('bytes/received', size(Data)), - received(Data, control_throttle(State #state{await_recv = false})); + emqttd_metrics:inc('bytes/received', Size), + received(Data, rate_limit(Size, State#state{await_recv = false})); handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> + %%TODO: ... network_error(Reason, State); +handle_info({inet_reply, _Ref, ok}, State) -> + %%TODO: ok... + io:format("inet_reply ok~n"), + noreply(State); + handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - ?ERROR("Unexpected inet_reply - ~p", [Reason], State), - {noreply, State}; + network_error(Reason, State); handle_info({keepalive, start, TimeoutSec}, State = #state{transport = Transport, socket = Socket}) -> ?DEBUG("Start KeepAlive with ~p seconds", [TimeoutSec], State), @@ -174,14 +184,14 @@ handle_info({keepalive, start, TimeoutSec}, State = #state{transport = Transport handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of - {ok, KeepAlive1} -> - noreply(State#state{keepalive = KeepAlive1}); - {error, timeout} -> - ?DEBUG("Keepalive Timeout!", [], State), - stop({shutdown, keepalive_timeout}, State#state{keepalive = undefined}); - {error, Error} -> - ?DEBUG("Keepalive Error - ~p", [Error], State), - stop({shutdown, keepalive_error}, State#state{keepalive = undefined}) + {ok, KeepAlive1} -> + noreply(State#state{keepalive = KeepAlive1}); + {error, timeout} -> + ?DEBUG("Keepalive Timeout!", [], State), + stop({shutdown, keepalive_timeout}, State#state{keepalive = undefined}); + {error, Error} -> + ?DEBUG("Keepalive Error - ~p", [Error], State), + stop({shutdown, keepalive_error}, State#state{keepalive = undefined}) end; handle_info(Info, State) -> @@ -223,27 +233,27 @@ with_session(Fun, State = #state{proto_state = ProtoState}) -> %% receive and parse tcp data received(<<>>, State) -> - {noreply, State, hibernate}; + noreply(State); -received(Bytes, State = #state{packet_opts = PacketOpts, - parser = Parser, +received(Bytes, State = #state{parser = Parser, + packet_opts = PacketOpts, proto_state = ProtoState}) -> case catch Parser(Bytes) of {more, NewParser} -> - noreply(control_throttle(State#state{parser = NewParser})); + noreply(run_socket(State#state{parser = NewParser})); {ok, Packet, Rest} -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of - {ok, ProtoState1} -> - received(Rest, State#state{parser = emqttd_parser:new(PacketOpts), - proto_state = ProtoState1}); - {error, Error} -> - ?ERROR("Protocol error - ~p", [Error], State), - stop({shutdown, Error}, State); - {error, Error, ProtoState1} -> - stop({shutdown, Error}, State#state{proto_state = ProtoState1}); - {stop, Reason, ProtoState1} -> - stop(Reason, State#state{proto_state = ProtoState1}) + {ok, ProtoState1} -> + received(Rest, State#state{parser = emqttd_parser:new(PacketOpts), + proto_state = ProtoState1}); + {error, Error} -> + ?ERROR("Protocol error - ~p", [Error], State), + stop({shutdown, Error}, State); + {error, Error, ProtoState1} -> + stop({shutdown, Error}, State#state{proto_state = ProtoState1}); + {stop, Reason, ProtoState1} -> + stop(Reason, State#state{proto_state = ProtoState1}) end; {error, Error} -> ?ERROR("Framing error - ~p", [Error], State), @@ -258,6 +268,20 @@ network_error(Reason, State = #state{peername = Peername}) -> [emqttd_net:format(Peername), Reason]), stop({shutdown, conn_closed}, State). +rate_limit(_Size, State = #state{rate_limiter = undefined}) -> + run_socket(State); +rate_limit(Size, State = #state{socket = Sock, rate_limiter = Limiter}) -> + {ok, BufSizes} = inet:getopts(Sock, [sndbuf, recbuf, buffer]), + io:format("~p~n", [BufSizes]), + case esockd_rate_limiter:check(Limiter, Size) of + {0, Limiter1} -> + run_socket(State#state{conn_state = running, rate_limiter = Limiter1}); + {Pause, Limiter1} -> + ?ERROR("~p Received, Rate Limiter Pause for ~w", [Size, Pause], State), + erlang:send_after(Pause, self(), activate_sock), + State#state{conn_state = blocked, rate_limiter = Limiter1} + end. + run_socket(State = #state{conn_state = blocked}) -> State; run_socket(State = #state{await_recv = true}) -> @@ -266,11 +290,11 @@ run_socket(State = #state{transport = Transport, socket = Sock}) -> Transport:async_recv(Sock, 0, infinity), State#state{await_recv = true}. -control_throttle(State = #state{conn_state = Flow, - conserve = Conserve}) -> - case {Flow, Conserve} of - {running, true} -> State #state{conn_state = blocked}; - {blocked, false} -> run_socket(State #state{conn_state = running}); - {_, _} -> run_socket(State) +send_fun(Transport, Sock) -> + fun(Data) -> + try Transport:port_command(Sock, Data) of + true -> ok + catch + error:Error -> exit({socket_error, Error}) + end end. - From 565c8abb3a9f85c235adf96ab9907e4c3358ec34 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Mon, 26 Oct 2015 16:30:39 +0800 Subject: [PATCH 11/37] improve suback --- src/emqttd_client.erl | 17 +++++++++++------ src/emqttd_protocol.erl | 5 +---- src/emqttd_session.erl | 12 ++++++++---- src/emqttd_ws_client.erl | 5 +++++ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index b8909ff32..247fe7500 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -135,12 +135,11 @@ handle_cast(Msg, State) -> handle_info(timeout, State) -> stop({shutdown, timeout}, State); - -handle_info({stop, duplicate_id, _NewPid}, State=#state{proto_state = ProtoState, - conn_name = ConnName}) -> - lager:warning("Shutdown for duplicate clientid: ~s, conn:~s", - [emqttd_protocol:clientid(ProtoState), ConnName]), - stop({shutdown, duplicate_id}, State); + +%% Asynchronous SUBACK +handle_info({suback, PacketId, GrantedQos}, State = #state{proto_state = ProtoState}) -> + {ok, ProtoState1} = emqttd_protocol:send(?SUBACK_PACKET(PacketId, GrantedQos), ProtoState), + noreply(State#state{proto_state = ProtoState1}); handle_info({deliver, Message}, State = #state{proto_state = ProtoState}) -> {ok, ProtoState1} = emqttd_protocol:send(Message, ProtoState), @@ -150,6 +149,12 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State = #state{proto_state = Prot {ok, ProtoState1} = emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState), noreply(State#state{proto_state = ProtoState1}); +handle_info({stop, duplicate_id, _NewPid}, State=#state{proto_state = ProtoState, + conn_name = ConnName}) -> + lager:warning("Shutdown for duplicate clientid: ~s, conn:~s", + [emqttd_protocol:clientid(ProtoState), ConnName]), + stop({shutdown, duplicate_id}, State); + handle_info(activate_sock, State) -> noreply(run_socket(State#state{conn_state = running})); diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index ce85ddab3..b4371f525 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -240,10 +240,7 @@ process(?SUBSCRIBE_PACKET(PacketId, TopicTable), lager:error("SUBSCRIBE from '~s' Denied: ~p", [ClientId, TopicTable]), send(?SUBACK_PACKET(PacketId, [16#80 || _ <- TopicTable]), State); false -> - AckFun = fun(GrantedQos) -> - send(?SUBACK_PACKET(PacketId, GrantedQos), State) - end, - emqttd_session:subscribe(Session, TopicTable, AckFun), {ok, State} + emqttd_session:subscribe(Session, PacketId, TopicTable), {ok, State} end; %% protect from empty topic list diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index f33e72521..12583fb5d 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -167,8 +167,12 @@ destroy(SessPid, ClientId) -> subscribe(SessPid, TopicTable) -> subscribe(SessPid, TopicTable, fun(_) -> ok end). --spec subscribe(pid(), [{binary(), mqtt_qos()}], AckFun :: fun()) -> ok. -subscribe(SessPid, TopicTable, AckFun) -> +-spec subscribe(pid(), mqtt_packet_id(), [{binary(), mqtt_qos()}]) -> ok. +subscribe(SessPid, PacketId, TopicTable) -> + From = self(), + AckFun = fun(GrantedQos) -> + From ! {suback, PacketId, GrantedQos} + end, gen_server2:cast(SessPid, {subscribe, TopicTable, AckFun}). %%------------------------------------------------------------------------------ @@ -298,13 +302,13 @@ handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{ case TopicTable -- Subscriptions of [] -> - catch AckFun([Qos || {_, Qos} <- TopicTable]), + AckFun([Qos || {_, Qos} <- TopicTable]), noreply(Session); _ -> %% subscribe first and don't care if the subscriptions have been existed {ok, GrantedQos} = emqttd_pubsub:subscribe(TopicTable), - catch AckFun(GrantedQos), + AckFun(GrantedQos), emqttd_broker:foreach_hooks('client.subscribe.after', [ClientId, TopicTable]), diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 4d72ce7bb..b4f82c043 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -164,6 +164,11 @@ handle_cast({received, Packet}, State = #client_state{proto_state = ProtoState}) handle_cast(_Msg, State) -> {noreply, State}. +%% Asynchronous SUBACK +handle_info({suback, PacketId, GrantedQos}, State = #client_state{proto_state = ProtoState}) -> + {ok, ProtoState1} = emqttd_protocol:send(?SUBACK_PACKET(PacketId, GrantedQos), ProtoState), + noreply(State#client_state{proto_state = ProtoState1}); + handle_info({deliver, Message}, State = #client_state{proto_state = ProtoState}) -> {ok, ProtoState1} = emqttd_protocol:send(Message, ProtoState), noreply(State#client_state{proto_state = ProtoState1}); From 4698856215c177274529a92cdb3d67190a588605 Mon Sep 17 00:00:00 2001 From: Feng Date: Thu, 29 Oct 2015 21:46:06 +0800 Subject: [PATCH 12/37] connopts, ratelimit --- rel/files/emqttd.config.development | 19 +++++++++++++++++-- rel/files/emqttd.config.production | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/rel/files/emqttd.config.development b/rel/files/emqttd.config.development index 6ec4754c6..16c475523 100644 --- a/rel/files/emqttd.config.development +++ b/rel/files/emqttd.config.development @@ -176,36 +176,50 @@ %% File to store loaded plugin names. {loaded_file, "./data/loaded_plugins"} ]}, + %% Listeners {listeners, [ {mqtt, 1883, [ %% Size of acceptor pool {acceptors, 16}, + %% Maximum number of concurrent clients {max_clients, 512}, + %% Socket Access Control {access, [{allow, all}]}, - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"}, %% 100K burst, 10K rate + + %% Connection Options + {connopts, [ + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec + %% {rate_limit, "100,10"} %% 100K burst, 10K rate + ]}, + %% Socket Options {sockopts, [ %Set buffer if hight thoughtput %{recbuf, 4096}, %{sndbuf, 4096}, %{buffer, 4096}, + %{nodelay, true}, {backlog, 512} ]} ]}, + {mqtts, 8883, [ %% Size of acceptor pool {acceptors, 4}, + %% Maximum number of concurrent clients {max_clients, 512}, + %% Socket Access Control {access, [{allow, all}]}, + %% SSL certificate and key files {ssl, [{certfile, "etc/ssl/ssl.crt"}, {keyfile, "etc/ssl/ssl.key"}]}, + %% Socket Options {sockopts, [ {backlog, 1024} @@ -229,6 +243,7 @@ %% {backlog, 1024} %% ]} %%]}, + %% HTTP and WebSocket Listener {http, 8083, [ %% Size of acceptor pool diff --git a/rel/files/emqttd.config.production b/rel/files/emqttd.config.production index e3685518f..a657d7bf3 100644 --- a/rel/files/emqttd.config.production +++ b/rel/files/emqttd.config.production @@ -185,7 +185,8 @@ %{recbuf, 4096}, %{sndbuf, 4096}, %{buffer, 4096}, - {backlog, 512} + %{nodelay, true}, + {backlog, 1024} ]} ]}, {mqtts, 8883, [ From 8d9270a6ba1e63bc50d355082e2e33fc83a2373d Mon Sep 17 00:00:00 2001 From: Feng Date: Thu, 29 Oct 2015 21:46:17 +0800 Subject: [PATCH 13/37] 48 --- rel/files/vm.args | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rel/files/vm.args b/rel/files/vm.args index 8e37830dc..9cc798802 100644 --- a/rel/files/vm.args +++ b/rel/files/vm.args @@ -17,7 +17,9 @@ ## Enable kernel poll and a few async threads +K true -+A 16 + +## 12 threads/core. ++A 48 ## max process numbers +P 8192 From f4cf90ae91d18538db261538c6c917e81fad11f3 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 30 Oct 2015 16:00:11 +0800 Subject: [PATCH 14/37] refactor client --- rebar.config | 4 +- src/emqttd_client.erl | 314 +++++++++++++++++++++--------------------- 2 files changed, 161 insertions(+), 157 deletions(-) diff --git a/rebar.config b/rebar.config index 9dcb12ee6..4b39a2f2b 100644 --- a/rebar.config +++ b/rebar.config @@ -29,8 +29,8 @@ {deps, [ {gproc, ".*", {git, "git://github.com/uwiger/gproc.git", {branch, "master"}}}, {lager, ".*", {git, "git://github.com/basho/lager.git", {branch, "master"}}}, - {esockd, "2.*", {git, "git://github.com/emqtt/esockd.git", {branch, "master"}}}, - {mochiweb, ".*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "master"}}} + {esockd, "3.*", {git, "git://github.com/emqtt/esockd.git", {branch, "3.0"}}}, + {mochiweb, ".*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "4.0"}}} ]}. {recursive_cmds, [ct, eunit, clean]}. diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 247fe7500..e8a143734 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_client). -author("Feng Lee "). @@ -33,40 +32,35 @@ -include("emqttd_protocol.hrl"). +-include("emqttd_internel.hrl"). + +-behaviour(gen_server). + %% API Function Exports -export([start_link/2, session/1, info/1, kick/1]). -%% SUB/UNSUB Asynchronously +%% SUB/UNSUB Asynchronously, called by plugins. -export([subscribe/2, unsubscribe/2]). --behaviour(gen_server). - %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]). -%% Client State... --record(state, {transport, - socket, - peername, - conn_name, - await_recv, - conn_state, - rate_limiter, - parser, - proto_state, - packet_opts, - keepalive}). +%% Client State +-record(client_state, {connection, peername, peerhost, peerport, + await_recv, conn_state, rate_limit, + parser_fun, proto_state, packet_opts, + keepalive}). --define(DEBUG(Format, Args, State), - lager:debug("Client(~s): " ++ Format, - [emqttd_net:format(State#state.peername) | Args])). --define(ERROR(Format, Args, State), - lager:error("Client(~s): " ++ Format, - [emqttd_net:format(State#state.peername) | Args])). +-define(INFO_KEYS, [peername, peerhost, peerport, await_recv, conn_state]). -start_link(SockArgs, MqttEnv) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[SockArgs, MqttEnv]])}. +-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). + +-define(LOG(Level, Format, Args, State), + lager:Level("Client(~s): " ++ Format, [State#client_state.peername | Args])). + +start_link(Connection, MqttEnv) -> + {ok, proc_lib:spawn_link(?MODULE, init, [[Connection, MqttEnv]])}. session(CPid) -> gen_server:call(CPid, session, infinity). @@ -83,140 +77,156 @@ subscribe(CPid, TopicTable) -> unsubscribe(CPid, Topics) -> gen_server:cast(CPid, {unsubscribe, Topics}). -init([SockArgs = {Transport, Sock, _SockFun}, MqttEnv]) -> - % Transform if ssl. - {ok, NewSock} = esockd_connection:accept(SockArgs), - %%TODO:... - {ok, BufSizes} = inet:getopts(Sock, [sndbuf, recbuf, buffer]), - io:format("~p~n", [BufSizes]), - {ok, Peername} = emqttd_net:peername(Sock), - {ok, ConnStr} = emqttd_net:connection_string(Sock, inbound), - SendFun = send_fun(Transport, NewSock), - PktOpts = proplists:get_value(packet, MqttEnv), - ProtoState = emqttd_protocol:init(Peername, SendFun, PktOpts), - Limiter = proplists:get_value(rate_limiter, MqttEnv), - State = run_socket(#state{transport = Transport, - socket = NewSock, - peername = Peername, - conn_name = ConnStr, - await_recv = false, - conn_state = running, - rate_limiter = Limiter, - packet_opts = PktOpts, - parser = emqttd_parser:new(PktOpts), - proto_state = ProtoState}), +init([Connection0, MqttEnv]) -> + {ok, Connection} = Connection0:wait(), + {PeerHost, PeerPort, PeerName} = + case Connection:peername() of + {ok, {Host, Port}} -> + {Host, Port, esockd_net:format({Host, Port})}; + {error, enotconn} -> + Connection:fast_close(), + exit(normal); + {error, Reason} -> + Connection:fast_close(), + exit({shutdown, Reason}) + end, + SendFun = fun(Data) -> + try Connection:async_send(Data) of + true -> ok + catch + error:Error -> exit({shutdown, Error}) + end + end, + PktOpts = proplists:get_value(packet, MqttEnv), + ParserFun = emqttd_parser:new(PktOpts), + ProtoState = emqttd_protocol:init(PeerName, SendFun, PktOpts), + RateLimit = proplists:get_value(rate_limit, Connection:opts()), + State = run_socket(#client_state{connection = Connection, + peername = PeerName, + peerhost = PeerHost, + peerport = PeerPort, + await_recv = false, + conn_state = running, + rate_limit = RateLimit, + parser_fun = ParserFun, + proto_state = ProtoState, + packet_opts = PktOpts}), ClientOpts = proplists:get_value(client, MqttEnv), IdleTimout = proplists:get_value(idle_timeout, ClientOpts, 10), gen_server:enter_loop(?MODULE, [], State, timer:seconds(IdleTimout)). -handle_call(session, _From, State = #state{proto_state = ProtoState}) -> +handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> {reply, emqttd_protocol:session(ProtoState), State}; -handle_call(info, _From, State = #state{conn_name = ConnName, - proto_state = ProtoState}) -> - {reply, [{conn_name, ConnName} | emqttd_protocol:info(ProtoState)], State}; +handle_call(info, _From, State = #client_state{connection = Connection, + proto_state = ProtoState}) -> + ClientInfo = [{Key, Val} || {Key, Val} <- ?record_to_proplist(client_state, State), lists:member(Key, ?INFO_KEYS)], + ProtoInfo = emqttd_protocol:info(ProtoState), + {ok, SockStats} = Connection:getstat(?SOCK_STATS), + Info = lists:append([ClientInfo, [{proto_info, ProtoInfo}, {sock_stats, SockStats}]]), + {reply, Info, State}; handle_call(kick, _From, State) -> {stop, {shutdown, kick}, ok, State}; handle_call(Req, _From, State) -> - ?ERROR("Unexpected request: ~p", [Req], State), - {reply, {error, unsupported_request}, State}. + ?LOG(critical, "Unexpected request: ~p", [Req], State), + {reply, {error, unsupported_request}, State}. handle_cast({subscribe, TopicTable}, State) -> - with_session(fun(SessPid) -> emqttd_session:subscribe(SessPid, TopicTable) end, State); + with_session(fun(SessPid) -> + emqttd_session:subscribe(SessPid, TopicTable) + end, State); handle_cast({unsubscribe, Topics}, State) -> - with_session(fun(SessPid) -> emqttd_session:unsubscribe(SessPid, Topics) end, State); + with_session(fun(SessPid) -> + emqttd_session:unsubscribe(SessPid, Topics) + end, State); handle_cast(Msg, State) -> - ?ERROR("Unexpected msg: ~p",[Msg], State), - {noreply, State}. + ?LOG(critical, "Unexpected msg: ~p", [Msg], State), + noreply(State). handle_info(timeout, State) -> - stop({shutdown, timeout}, State); + shutdown(idle_timeout, State); %% Asynchronous SUBACK -handle_info({suback, PacketId, GrantedQos}, State = #state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:send(?SUBACK_PACKET(PacketId, GrantedQos), ProtoState), - noreply(State#state{proto_state = ProtoState1}); +handle_info({suback, PacketId, GrantedQos}, State) -> + with_proto_state(fun(ProtoState) -> + Packet = ?SUBACK_PACKET(PacketId, GrantedQos), + emqttd_protocol:send(Packet, ProtoState) + end, State); -handle_info({deliver, Message}, State = #state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:send(Message, ProtoState), - noreply(State#state{proto_state = ProtoState1}); +handle_info({deliver, Message}, State) -> + with_proto_state(fun(ProtoState) -> + emqttd_protocol:send(Message, ProtoState) + end, State); -handle_info({redeliver, {?PUBREL, PacketId}}, State = #state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState), - noreply(State#state{proto_state = ProtoState1}); +handle_info({redeliver, {?PUBREL, PacketId}}, State) -> + with_proto_state(fun(ProtoState) -> + emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState) + end, State); -handle_info({stop, duplicate_id, _NewPid}, State=#state{proto_state = ProtoState, - conn_name = ConnName}) -> - lager:warning("Shutdown for duplicate clientid: ~s, conn:~s", - [emqttd_protocol:clientid(ProtoState), ConnName]), - stop({shutdown, duplicate_id}, State); +handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> + ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State), + shutdown(confict, State); handle_info(activate_sock, State) -> - noreply(run_socket(State#state{conn_state = running})); + noreply(run_socket(State#client_state{conn_state = running})); -handle_info({inet_async, Sock, _Ref, {ok, Data}}, State = #state{peername = Peername, socket = Sock}) -> +handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> Size = size(Data), - lager:debug("RECV from ~s: ~p", [emqttd_net:format(Peername), Data]), + ?LOG(debug, "RECV: ~p", [Data], State), emqttd_metrics:inc('bytes/received', Size), - received(Data, rate_limit(Size, State#state{await_recv = false})); + received(Data, rate_limit(Size, State#client_state{await_recv = false})); handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> - %%TODO: ... - network_error(Reason, State); + shutdown(Reason, State); -handle_info({inet_reply, _Ref, ok}, State) -> - %%TODO: ok... - io:format("inet_reply ok~n"), +handle_info({inet_reply, _Sock, ok}, State) -> noreply(State); handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - network_error(Reason, State); + shutdown(Reason, State); -handle_info({keepalive, start, TimeoutSec}, State = #state{transport = Transport, socket = Socket}) -> - ?DEBUG("Start KeepAlive with ~p seconds", [TimeoutSec], State), +handle_info({keepalive, start, Interval}, State = #client_state{connection = Connection}) -> + ?LOG(debug, "Keepalive at the interval of ~p", [Interval], State), StatFun = fun() -> - case Transport:getstat(Socket, [recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; - {error, Error} -> {error, Error} - end - end, - KeepAlive = emqttd_keepalive:start(StatFun, TimeoutSec, {keepalive, check}), - noreply(State#state{keepalive = KeepAlive}); + case Connection:getstat([recv_oct]) of + {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; + {error, Error} -> {error, Error} + end + end, + KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}), + noreply(State#client_state{keepalive = KeepAlive}); -handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> +handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> noreply(State#state{keepalive = KeepAlive1}); {error, timeout} -> - ?DEBUG("Keepalive Timeout!", [], State), - stop({shutdown, keepalive_timeout}, State#state{keepalive = undefined}); + ?LOG(debug, "Keepalive timeout", [], State), + shutdown(keepalive_timeout, State); {error, Error} -> - ?DEBUG("Keepalive Error - ~p", [Error], State), - stop({shutdown, keepalive_error}, State#state{keepalive = undefined}) + ?LOG(warning, "Keepalive error - ~p", [Error], State), + shutdown(Error, State) end; handle_info(Info, State) -> - ?ERROR("Unexpected info: ~p", [Info], State), - {noreply, State}. + ?LOG(critical, "Unexpected info: ~p", [Info], State), + noreply(State). -terminate(Reason, #state{transport = Transport, - socket = Socket, - keepalive = KeepAlive, - proto_state = ProtoState}) -> +terminate(Reason, #client_state{connection = Connection, + keepalive = KeepAlive, + proto_state = ProtoState}) -> + Connection:fast_close(), emqttd_keepalive:cancel(KeepAlive), - if - Reason == {shutdown, conn_closed} -> ok; - true -> Transport:fast_close(Socket) - end, case {ProtoState, Reason} of - {undefined, _} -> ok; + {undefined, _} -> + ok; {_, {shutdown, Error}} -> emqttd_protocol:shutdown(Error, ProtoState); - {_, Reason} -> + {_, Reason} -> emqttd_protocol:shutdown(Reason, ProtoState) end. @@ -227,79 +237,73 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================= -noreply(State) -> - {noreply, State, hibernate}. +with_proto_state(Fun, State = #client_state{proto_state = ProtoState}) -> + {ok, ProtoState1} = Fun(ProtoState), + noreply(State#client_state{proto_state = ProtoState1}). -stop(Reason, State) -> - {stop, Reason, State}. - -with_session(Fun, State = #state{proto_state = ProtoState}) -> - Fun(emqttd_protocol:session(ProtoState)), noreply(State). +with_session(Fun, State = #client_state{proto_state = ProtoState}) -> + Fun(emqttd_protocol:session(ProtoState)), + noreply(State). %% receive and parse tcp data received(<<>>, State) -> noreply(State); -received(Bytes, State = #state{parser = Parser, - packet_opts = PacketOpts, - proto_state = ProtoState}) -> - case catch Parser(Bytes) of +received(Bytes, State = #client_state{parser_fun = ParserFun, + packet_opts = PacketOpts, + proto_state = ProtoState}) -> + case catch ParserFun(Bytes) of {more, NewParser} -> - noreply(run_socket(State#state{parser = NewParser})); + noreply(run_socket(State#client_state{parser_fun = NewParser})); {ok, Packet, Rest} -> emqttd_metrics:received(Packet), case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - received(Rest, State#state{parser = emqttd_parser:new(PacketOpts), - proto_state = ProtoState1}); + received(Rest, State#client_state{parser_fun = emqttd_parser:new(PacketOpts), + proto_state = ProtoState1}); {error, Error} -> - ?ERROR("Protocol error - ~p", [Error], State), - stop({shutdown, Error}, State); + ?LOG(error, "Protocol error - ~p", [Error], State), + shutdown(Error, State); {error, Error, ProtoState1} -> - stop({shutdown, Error}, State#state{proto_state = ProtoState1}); + shutdown(Error, State#client_state{proto_state = ProtoState1}); {stop, Reason, ProtoState1} -> - stop(Reason, State#state{proto_state = ProtoState1}) + stop(Reason, State#client_state{proto_state = ProtoState1}) end; {error, Error} -> - ?ERROR("Framing error - ~p", [Error], State), - stop({shutdown, Error}, State); + ?LOG(error, "Framing error - ~p", [Error], State), + shutdown(Error, State); {'EXIT', Reason} -> - ?ERROR("Parser failed for ~p~nError Frame: ~p", [Reason, Bytes], State), - {stop, {shutdown, frame_error}, State} + ?LOG(error, "Parser failed for ~p", [Reason], State), + ?LOG(error, "Error data: ~p", [Bytes], State), + shutdown(parser_error, State) end. -network_error(Reason, State = #state{peername = Peername}) -> - lager:warning("Client(~s): network error - ~p", - [emqttd_net:format(Peername), Reason]), - stop({shutdown, conn_closed}, State). - -rate_limit(_Size, State = #state{rate_limiter = undefined}) -> +rate_limit(_Size, State = #client_state{rate_limit = undefined}) -> run_socket(State); -rate_limit(Size, State = #state{socket = Sock, rate_limiter = Limiter}) -> - {ok, BufSizes} = inet:getopts(Sock, [sndbuf, recbuf, buffer]), - io:format("~p~n", [BufSizes]), - case esockd_rate_limiter:check(Limiter, Size) of +rate_limit(Size, State = #client_state{rate_limit = Limiter}) -> + case esockd_ratelimit:check(Limiter, Size) of {0, Limiter1} -> - run_socket(State#state{conn_state = running, rate_limiter = Limiter1}); + run_socket(State#client_state{conn_state = running, rate_limit = Limiter1}); {Pause, Limiter1} -> - ?ERROR("~p Received, Rate Limiter Pause for ~w", [Size, Pause], State), + ?LOG(error, "Rate limiter pause for ~p", [Size, Pause], State), erlang:send_after(Pause, self(), activate_sock), - State#state{conn_state = blocked, rate_limiter = Limiter1} + State#client_state{conn_state = blocked, rate_limit = Limiter1} end. -run_socket(State = #state{conn_state = blocked}) -> +run_socket(State = #client_state{conn_state = blocked}) -> State; -run_socket(State = #state{await_recv = true}) -> +run_socket(State = #client_state{await_recv = true}) -> State; -run_socket(State = #state{transport = Transport, socket = Sock}) -> - Transport:async_recv(Sock, 0, infinity), - State#state{await_recv = true}. +run_socket(State = #client_state{connection = Connection}) -> + Connection:async_recv(0, infinity), + State#client_state{await_recv = true}. + +noreply(State) -> + {noreply, State, hibernate}. + +shutdown(Reason, State) -> + stop({shutdown, Reason}, State). + +stop(Reason, State) -> + {stop, Reason, State}. -send_fun(Transport, Sock) -> - fun(Data) -> - try Transport:port_command(Sock, Data) of - true -> ok - catch - error:Error -> exit({socket_error, Error}) - end - end. From f6636a978384e82f3e8025d149665405e34ba5f6 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 30 Oct 2015 16:03:25 +0800 Subject: [PATCH 15/37] refactor client --- include/emqttd_internal.hrl | 31 +++++++++++++++++++++++++++++++ src/emqttd_client.erl | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 include/emqttd_internal.hrl diff --git a/include/emqttd_internal.hrl b/include/emqttd_internal.hrl new file mode 100644 index 000000000..fa8440e45 --- /dev/null +++ b/include/emqttd_internal.hrl @@ -0,0 +1,31 @@ +%%%----------------------------------------------------------------------------- +%%% Copyright (c) 2012-2015 eMQTT.IO, All Rights Reserved. +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in all +%%% copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +%%% SOFTWARE. +%%%----------------------------------------------------------------------------- +%%% @doc +%%% MQTT Internal Header. +%%% +%%% @end +%%%----------------------------------------------------------------------------- + +-define(record_to_proplist(Def, Rec), + lists:zip(record_info(fields, Def), + tl(tuple_to_list(Rec)))). + diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index e8a143734..88f71b3cd 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -32,7 +32,7 @@ -include("emqttd_protocol.hrl"). --include("emqttd_internel.hrl"). +-include("emqttd_internal.hrl"). -behaviour(gen_server). From 1ef861715e425d61b0efd39c701f3ebb0b6cc327 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 30 Oct 2015 16:16:56 +0800 Subject: [PATCH 16/37] rm sockjs --- plugins/emqttd_sockjs | 1 - 1 file changed, 1 deletion(-) delete mode 160000 plugins/emqttd_sockjs diff --git a/plugins/emqttd_sockjs b/plugins/emqttd_sockjs deleted file mode 160000 index 6d5ba0dfe..000000000 --- a/plugins/emqttd_sockjs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6d5ba0dfe62d375da09f1d53823b8aa54046aa11 From 19930b6382a9eb6789af21ef5060fd7409007a59 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 30 Oct 2015 18:00:23 +0800 Subject: [PATCH 17/37] refactor protocol --- src/emqttd_client.erl | 12 +-- src/emqttd_protocol.erl | 207 +++++++++++++++++----------------------- 2 files changed, 95 insertions(+), 124 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 88f71b3cd..125ac40df 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -120,11 +120,11 @@ handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> handle_call(info, _From, State = #client_state{connection = Connection, proto_state = ProtoState}) -> - ClientInfo = [{Key, Val} || {Key, Val} <- ?record_to_proplist(client_state, State), lists:member(Key, ?INFO_KEYS)], - ProtoInfo = emqttd_protocol:info(ProtoState), + ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS), + ProtoInfo = emqttd_protocol:info(ProtoState), {ok, SockStats} = Connection:getstat(?SOCK_STATS), - Info = lists:append([ClientInfo, [{proto_info, ProtoInfo}, {sock_stats, SockStats}]]), - {reply, Info, State}; + {noreply, lists:append([ClientInfo, [{proto_info, ProtoInfo}, + {sock_stats, SockStats}]]), State}; handle_call(kick, _From, State) -> {stop, {shutdown, kick}, ok, State}; @@ -176,7 +176,7 @@ handle_info(activate_sock, State) -> handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> Size = size(Data), - ?LOG(debug, "RECV: ~p", [Data], State), + ?LOG(debug, "RECV <- ~p", [Data], State), emqttd_metrics:inc('bytes/received', Size), received(Data, rate_limit(Size, State#client_state{await_recv = false})); @@ -203,7 +203,7 @@ handle_info({keepalive, start, Interval}, State = #client_state{connection = Con handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> - noreply(State#state{keepalive = KeepAlive1}); + noreply(State#client_state{keepalive = KeepAlive1}); {error, timeout} -> ?LOG(debug, "Keepalive timeout", [], State), shutdown(keepalive_timeout, State); diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index b4371f525..a2311ec7d 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_protocol). -author("Feng Lee "). @@ -33,6 +32,8 @@ -include("emqttd_protocol.hrl"). +-include("emqttd_internal.hrl"). + %% API -export([init/3, info/1, clientid/1, client/1, session/1]). @@ -41,29 +42,26 @@ -export([process/2]). %% Protocol State --record(proto_state, {peername, - sendfun, - connected = false, %received CONNECT action? - proto_ver, - proto_name, - username, - client_id, - clean_sess, - session, - will_msg, - keepalive, - max_clientid_len = ?MAX_CLIENTID_LEN, - client_pid, - ws_initial_headers, %% Headers from first HTTP request for websocket client +-record(proto_state, {peername, sendfun, connected = false, + client_id, client_pid, clean_sess, + proto_ver, proto_name, username, + will_msg, keepalive, max_clientid_len = ?MAX_CLIENTID_LEN, + session, ws_initial_headers, %% Headers from first HTTP request for websocket client connected_at}). -type proto_state() :: #proto_state{}. +-define(INFO_KEYS, [client_id, username, clean_sess, proto_ver, proto_name, + keepalive, will_msg, ws_initial_headers, connected_at]). + +-define(LOG(Level, Format, Args, State), + lager:Level([{client, State#proto_state.client_id}], "Client(~s@~s): " ++ Format, + [State#proto_state.client_id, State#proto_state.peername | Args])). + %%------------------------------------------------------------------------------ %% @doc Init protocol %% @end %%------------------------------------------------------------------------------ - init(Peername, SendFun, Opts) -> MaxLen = emqttd_opts:g(max_clientid_len, Opts, ?MAX_CLIENTID_LEN), WsInitialHeaders = emqttd_opts:g(ws_initial_headers, Opts), @@ -73,38 +71,20 @@ init(Peername, SendFun, Opts) -> client_pid = self(), ws_initial_headers = WsInitialHeaders}. -info(#proto_state{client_id = ClientId, - username = Username, - peername = Peername, - proto_ver = ProtoVer, - proto_name = ProtoName, - keepalive = KeepAlive, - clean_sess = CleanSess, - ws_initial_headers = WsInitialHeaders, - will_msg = WillMsg, - connected_at = ConnectedAt}) -> - [{client_id, ClientId}, - {username, Username}, - {peername, Peername}, - {proto_ver, ProtoVer}, - {proto_name, ProtoName}, - {keepalive, KeepAlive}, - {clean_sess, CleanSess}, - {ws_initial_headers, WsInitialHeaders}, - {will_msg, WillMsg}, - {connected_at, ConnectedAt}]. +info(ProtoState) -> + ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS). clientid(#proto_state{client_id = ClientId}) -> ClientId. client(#proto_state{client_id = ClientId, + client_pid = ClientPid, peername = Peername, username = Username, clean_sess = CleanSess, proto_ver = ProtoVer, keepalive = Keepalive, will_msg = WillMsg, - client_pid = Pid, ws_initial_headers = WsInitialHeaders, connected_at = Time}) -> WillTopic = if @@ -112,7 +92,7 @@ client(#proto_state{client_id = ClientId, true -> WillMsg#mqtt_message.topic end, #mqtt_client{client_id = ClientId, - client_pid = Pid, + client_pid = ClientPid, username = Username, peername = Peername, clean_sess = CleanSess, @@ -127,7 +107,7 @@ session(#proto_state{session = Session}) -> %% CONNECT – Client requests a connection to a Server -%%A Client can only send the CONNECT Packet once over a Network Connection. +%% A Client can only send the CONNECT Packet once over a Network Connection. -spec received(mqtt_packet(), proto_state()) -> {ok, proto_state()} | {error, any()}. received(Packet = ?PACKET(?CONNECT), State = #proto_state{connected = false}) -> process(Packet, State#proto_state{connected = true}); @@ -135,20 +115,20 @@ received(Packet = ?PACKET(?CONNECT), State = #proto_state{connected = false}) -> received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> {error, protocol_bad_connect, State}; -%%Received other packets when CONNECT not arrived. +%% Received other packets when CONNECT not arrived. received(_Packet, State = #proto_state{connected = false}) -> {error, protocol_not_connected, State}; received(Packet = ?PACKET(_Type), State) -> trace(recv, Packet, State), case validate_packet(Packet) of - ok -> - process(Packet, State); - {error, Reason} -> - {error, Reason, State} + ok -> + process(Packet, State); + {error, Reason} -> + {error, Reason, State} end. -process(Packet = ?CONNECT_PACKET(Var), State0 = #proto_state{peername = Peername}) -> +process(Packet = ?CONNECT_PACKET(Var), State0) -> #mqtt_packet_connect{proto_ver = ProtoVer, proto_name = ProtoName, @@ -190,10 +170,8 @@ process(Packet = ?CONNECT_PACKET(Var), State0 = #proto_state{peername = Peername exit({shutdown, Error}) end; {error, Reason}-> - lager:error("~s@~s: username '~s' login failed for ~s", - [ClientId, emqttd_net:format(Peername), Username, Reason]), + ?LOG(error, "Username '~s' login failed for ~s", [Username, Reason], State1), {?CONNACK_CREDENTIALS, State1} - end; ReturnCode -> {ReturnCode, State1} @@ -203,19 +181,18 @@ process(Packet = ?CONNECT_PACKET(Var), State0 = #proto_state{peername = Peername %% Send connack send(?CONNACK_PACKET(ReturnCode1), State3); -process(Packet = ?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload), - State = #proto_state{client_id = ClientId}) -> - +process(Packet = ?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload), State) -> case check_acl(publish, Topic, State) of allow -> publish(Packet, State); - deny -> - lager:error("ACL Deny: ~s cannot publish to ~s", [ClientId, Topic]) + deny -> + ?LOG(error, "Cannot publish to ~s for ACL Deny", [Topic], State) end, {ok, State}; process(?PUBACK_PACKET(?PUBACK, PacketId), State = #proto_state{session = Session}) -> - emqttd_session:puback(Session, PacketId), {ok, State}; + emqttd_session:puback(Session, PacketId), + {ok, State}; process(?PUBACK_PACKET(?PUBREC, PacketId), State = #proto_state{session = Session}) -> emqttd_session:pubrec(Session, PacketId), @@ -228,22 +205,21 @@ process(?PUBACK_PACKET(?PUBREL, PacketId), State = #proto_state{session = Sessio process(?PUBACK_PACKET(?PUBCOMP, PacketId), State = #proto_state{session = Session})-> emqttd_session:pubcomp(Session, PacketId), {ok, State}; -%% protect from empty topic list +%% Protect from empty topic table process(?SUBSCRIBE_PACKET(PacketId, []), State) -> send(?SUBACK_PACKET(PacketId, []), State); -process(?SUBSCRIBE_PACKET(PacketId, TopicTable), - State = #proto_state{client_id = ClientId, session = Session}) -> +process(?SUBSCRIBE_PACKET(PacketId, TopicTable), State = #proto_state{session = Session}) -> AllowDenies = [check_acl(subscribe, Topic, State) || {Topic, _Qos} <- TopicTable], case lists:member(deny, AllowDenies) of true -> - lager:error("SUBSCRIBE from '~s' Denied: ~p", [ClientId, TopicTable]), + ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicTable], State), send(?SUBACK_PACKET(PacketId, [16#80 || _ <- TopicTable]), State); false -> emqttd_session:subscribe(Session, PacketId, TopicTable), {ok, State} end; -%% protect from empty topic list +%% Protect from empty topic list process(?UNSUBSCRIBE_PACKET(PacketId, []), State) -> send(?UNSUBACK_PACKET(PacketId), State); @@ -255,72 +231,65 @@ process(?PACKET(?PINGREQ), State) -> send(?PACKET(?PINGRESP), State); process(?PACKET(?DISCONNECT), State) -> - % clean willmsg + % Clean willmsg {stop, normal, State#proto_state{will_msg = undefined}}. publish(Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId), #proto_state{client_id = ClientId, session = Session}) -> - Msg = emqttd_message:from_packet(ClientId, Packet), - emqttd_session:publish(Session, Msg); + emqttd_session:publish(Session, emqttd_message:from_packet(ClientId, Packet)); -publish(Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), - State = #proto_state{client_id = ClientId, session = Session}) -> +publish(Packet = ?PUBLISH_PACKET(?QOS_1, _PacketId), State) -> + with_puback(?PUBACK, Packet, State); + +publish(Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> + with_puback(?PUBREC, Packet, State). + +with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId), + State = #proto_state{client_id = ClientId, session = Session}) -> Msg = emqttd_message:from_packet(ClientId, Packet), case emqttd_session:publish(Session, Msg) of ok -> - send(?PUBACK_PACKET(?PUBACK, PacketId), State); + send(?PUBACK_PACKET(Type, PacketId), State); {error, Error} -> - lager:error("Client(~s): publish qos1 error - ~p", [ClientId, Error]) - end; - -publish(Packet = ?PUBLISH_PACKET(?QOS_2, PacketId), - State = #proto_state{client_id = ClientId, session = Session}) -> - Msg = emqttd_message:from_packet(ClientId, Packet), - case emqttd_session:publish(Session, Msg) of - ok -> - send(?PUBACK_PACKET(?PUBREC, PacketId), State); - {error, Error} -> - lager:error("Client(~s): publish qos2 error - ~p", [ClientId, Error]) + ?LOG(error, "PUBLISH ~p error: ~p", [PacketId, Error], State) end. -spec send(mqtt_message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}. send(Msg, State) when is_record(Msg, mqtt_message) -> send(emqttd_message:to_packet(Msg), State); -send(Packet, State = #proto_state{sendfun = SendFun, peername = Peername}) +send(Packet, State = #proto_state{sendfun = SendFun}) when is_record(Packet, mqtt_packet) -> trace(send, Packet, State), emqttd_metrics:sent(Packet), Data = emqttd_serialiser:serialise(Packet), - lager:debug("SENT to ~s: ~p", [emqttd_net:format(Peername), Data]), + ?LOG(debug, "SENT -> ~p", [Data], State), emqttd_metrics:inc('bytes/sent', size(Data)), SendFun(Data), {ok, State}. -trace(recv, Packet, #proto_state{peername = Peername, client_id = ClientId}) -> - lager:info([{client, ClientId}], "RECV from ~s@~s: ~s", - [ClientId, emqttd_net:format(Peername), emqttd_packet:format(Packet)]); +trace(recv, Packet, ProtoState) -> + trace2("RECV <-", Packet, ProtoState); -trace(send, Packet, #proto_state{peername = Peername, client_id = ClientId}) -> - lager:info([{client, ClientId}], "SEND to ~s@~s: ~s", - [ClientId, emqttd_net:format(Peername), emqttd_packet:format(Packet)]). +trace(send, Packet, ProtoState) -> + trace2("SEND ->", Packet, ProtoState). + +trace2(Tag, Packet, #proto_state{peername = Peername, client_id = ClientId}) -> + lager:info([{client, ClientId}], "Client(~s@~s): ~s ~s", + [ClientId, Peername, Tag, emqttd_packet:format(Packet)]). %% @doc redeliver PUBREL PacketId redeliver({?PUBREL, PacketId}, State) -> send(?PUBREL_PACKET(PacketId), State). -shutdown(Error, #proto_state{client_id = undefined}) -> - lager:info("Protocol shutdown ~p", [Error]), +shutdown(_Error, #proto_state{client_id = undefined}) -> ignore; -shutdown(duplicate_id, #proto_state{client_id = ClientId}) -> - %% unregister the device +shutdown(confict, #proto_state{client_id = ClientId}) -> emqttd_cm:unregister(ClientId); -%% TODO: ClientId?? -shutdown(Error, #proto_state{peername = Peername, client_id = ClientId, will_msg = WillMsg}) -> - lager:info([{client, ClientId}], "Client ~s@~s: shutdown ~p", - [ClientId, emqttd_net:format(Peername), Error]), +shutdown(Error, State = #proto_state{client_id = ClientId, will_msg = WillMsg}) -> + ?LOG(info, "shutdown for ~p", [Error], State), send_willmsg(ClientId, WillMsg), emqttd_broker:foreach_hooks('client.disconnected', [Error, ClientId]), emqttd_cm:unregister(ClientId). @@ -341,7 +310,6 @@ maybe_set_clientid(State) -> send_willmsg(_ClientId, undefined) -> ignore; send_willmsg(ClientId, WillMsg) -> - lager:info("Client ~s send willmsg: ~p", [ClientId, WillMsg]), emqttd_pubsub:publish(WillMsg#mqtt_message{from = ClientId}). start_keepalive(0) -> ignore; @@ -368,52 +336,55 @@ validate_connect(Connect = #mqtt_packet_connect{}, ProtoState) -> validate_protocol(#mqtt_packet_connect{proto_ver = Ver, proto_name = Name}) -> lists:member({Ver, Name}, ?PROTOCOL_NAMES). -validate_clientid(#mqtt_packet_connect{client_id = ClientId}, #proto_state{max_clientid_len = MaxLen}) - when ( size(ClientId) >= 1 ) andalso ( size(ClientId) =< MaxLen ) -> +validate_clientid(#mqtt_packet_connect{client_id = ClientId}, + #proto_state{max_clientid_len = MaxLen}) + when (size(ClientId) >= 1) andalso (size(ClientId) =< MaxLen) -> true; %% MQTT3.1.1 allow null clientId. validate_clientid(#mqtt_packet_connect{proto_ver =?MQTT_PROTO_V311, - client_id = ClientId}, _ProtoState) + client_id = ClientId}, _ProtoState) when size(ClientId) =:= 0 -> true; -validate_clientid(#mqtt_packet_connect{proto_ver = Ver, - clean_sess = CleanSess, - client_id = ClientId}, _ProtoState) -> - lager:warning("Invalid ClientId: ~s, ProtoVer: ~p, CleanSess: ~s", [ClientId, Ver, CleanSess]), +validate_clientid(#mqtt_packet_connect{proto_ver = ProtoVer, + clean_sess = CleanSess}, ProtoState) -> + ?LOG(warning, "Invalid clientId. ProtoVer: ~p, CleanSess: ~s", + [ProtoVer, CleanSess], ProtoState), false. -validate_packet(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH}, - variable = #mqtt_packet_publish{topic_name = Topic}}) -> +validate_packet(?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload)) -> case emqttd_topic:validate({name, Topic}) of true -> ok; - false -> lager:warning("Error publish topic: ~p", [Topic]), {error, badtopic} + false -> {error, badtopic} end; -validate_packet(#mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE}, - variable = #mqtt_packet_subscribe{topic_table = Topics}}) -> - - validate_topics(filter, Topics); - -validate_packet(#mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE}, - variable = #mqtt_packet_subscribe{topic_table = Topics}}) -> +validate_packet(?SUBSCRIBE_PACKET(_PacketId, TopicTable)) -> + validate_topics(filter, TopicTable); +validate_packet(?UNSUBSCRIBE_PACKET(_PacketId, Topics)) -> validate_topics(filter, Topics); validate_packet(_Packet) -> ok. -validate_topics(Type, []) when Type =:= name orelse Type =:= filter -> - lager:error("Empty Topics!"), +validate_topics(_Type, []) -> {error, empty_topics}; -validate_topics(Type, Topics) when Type =:= name orelse Type =:= filter -> - ErrTopics = [Topic || {Topic, Qos} <- Topics, - not (emqttd_topic:validate({Type, Topic}) and validate_qos(Qos))], - case ErrTopics of +validate_topics(Type, TopicTable = [{_Topic, _Qos}|_]) + when Type =:= name orelse Type =:= filter -> + Valid = fun(Topic, Qos) -> + emqttd_topic:validate({Type, Topic}) and validate_qos(Qos) + end, + case [Topic || {Topic, Qos} <- TopicTable, not Valid(Topic, Qos)] of [] -> ok; - _ -> lager:error("Error Topics: ~p", [ErrTopics]), {error, badtopic} + _ -> {error, badtopic} + end; + +validate_topics(Type, Topics = [Topic0|_]) when is_binary(Topic0) -> + case [Topic || Topic <- Topics, not emqttd_topic:validate({Type, Topic})] of + [] -> ok; + _ -> {error, badtopic} end. validate_qos(undefined) -> @@ -423,7 +394,7 @@ validate_qos(Qos) when ?IS_QOS(Qos) -> validate_qos(_) -> false. -%% publish ACL is cached in process dictionary. +%% PUBLISH ACL is cached in process dictionary. check_acl(publish, Topic, State) -> case get({acl, publish, Topic}) of undefined -> From 4ee2abdbcfc577a30af636378e56034cb9c5dd17 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 30 Oct 2015 18:00:36 +0800 Subject: [PATCH 18/37] record_to_proplist/3 --- include/emqttd_internal.hrl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/emqttd_internal.hrl b/include/emqttd_internal.hrl index fa8440e45..4f2eb378c 100644 --- a/include/emqttd_internal.hrl +++ b/include/emqttd_internal.hrl @@ -29,3 +29,7 @@ lists:zip(record_info(fields, Def), tl(tuple_to_list(Rec)))). +-define(record_to_proplist(Def, Rec, Fields), + [{K, V} || {K, V} <- ?record_to_proplist(Def, Rec), + lists:member(K, Fields)]). + From 8d3b7b4c246a7e527e0d69290b52f31d79b9ef04 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 30 Oct 2015 21:19:35 +0800 Subject: [PATCH 19/37] client monitor --- include/emqttd.hrl | 1 + 1 file changed, 1 insertion(+) diff --git a/include/emqttd.hrl b/include/emqttd.hrl index 51bce2f6d..ab4b6ee72 100644 --- a/include/emqttd.hrl +++ b/include/emqttd.hrl @@ -91,6 +91,7 @@ -record(mqtt_client, { client_id :: binary() | undefined, client_pid :: pid(), + client_mon :: reference(), username :: binary() | undefined, peername :: {inet:ip_address(), integer()}, clean_sess :: boolean(), From 4ed62d018e4cc7566da9fc5af5a627b742e41ce0 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 30 Oct 2015 21:20:37 +0800 Subject: [PATCH 20/37] refactor log --- src/emqttd_cm.erl | 21 +++-- src/emqttd_pubsub.erl | 4 +- src/emqttd_session.erl | 208 +++++++++++++++++++++-------------------- src/emqttd_sm.erl | 36 +++---- 4 files changed, 136 insertions(+), 133 deletions(-) diff --git a/src/emqttd_cm.erl b/src/emqttd_cm.erl index db198c176..9f5462b57 100644 --- a/src/emqttd_cm.erl +++ b/src/emqttd_cm.erl @@ -37,8 +37,6 @@ -behaviour(gen_server2). --define(SERVER, ?MODULE). - %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -47,6 +45,9 @@ -define(CM_POOL, ?MODULE). +-define(LOG(Level, Format, Args, Client), + lager:Level("CM(~s): " ++ Format, [Client#mqtt_client.client_id|Args])). + %%%============================================================================= %%% API %%%============================================================================= @@ -102,15 +103,16 @@ init([Id, StatsFun]) -> handle_call(Req, _From, State) -> lager:error("unexpected request: ~p", [Req]), - {reply, {error, badreq}, State}. + {reply, {error, unsupported_req}, State}. -handle_cast({register, Client = #mqtt_client{client_id = ClientId, client_pid = Pid}}, State) -> +handle_cast({register, Client = #mqtt_client{client_id = ClientId, + client_pid = Pid}}, State) -> case ets:lookup(mqtt_client, ClientId) of [#mqtt_client{client_pid = Pid}] -> - lager:error("ClientId '~s' has been registered with ~p", [ClientId, Pid]), ignore; [#mqtt_client{client_pid = OldPid}] -> - lager:warning("ClientId '~s' is duplicated: pid=~p, oldpid=~p", [ClientId, Pid, OldPid]); + %% TODO: should cancel monitor + ?LOG(warning, "client ~p conflict with ~p", [Pid, OldPid], Client); [] -> ok end, @@ -121,10 +123,10 @@ handle_cast({unregister, ClientId, Pid}, State) -> case ets:lookup(mqtt_client, ClientId) of [#mqtt_client{client_pid = Pid}] -> ets:delete(mqtt_client, ClientId); - [_] -> + [_] -> ignore; [] -> - lager:error("Cannot find clientId '~s' with ~p", [ClientId, Pid]) + ?LOG(error, "Cannot find registered: ~p", [Pid], State) end, {noreply, setstats(State)}; @@ -137,7 +139,8 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #state{id = Id}) -> - gproc_pool:disconnect_worker(?CM_POOL, {?MODULE, Id}), ok. + gproc_pool:disconnect_worker(?CM_POOL, {?MODULE, Id}), + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/src/emqttd_pubsub.erl b/src/emqttd_pubsub.erl index 478db721d..decb23d1f 100644 --- a/src/emqttd_pubsub.erl +++ b/src/emqttd_pubsub.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_pubsub). -author("Feng Lee "). @@ -48,8 +47,7 @@ publish/1]). %% Local node --export([dispatch/2, - match/1]). +-export([dispatch/2, match/1]). -behaviour(gen_server2). diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index 12583fb5d..a1b4ab47a 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -44,7 +44,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_session). -author("Feng Lee "). @@ -53,16 +52,15 @@ -include("emqttd_protocol.hrl"). +-behaviour(gen_server2). + %% Session API --export([start_link/3, resume/3, destroy/2]). +-export([start_link/3, resume/3, info/1, destroy/2]). %% PubSub APIs --export([publish/2, - puback/2, pubrec/2, pubrel/2, pubcomp/2, +-export([publish/2, puback/2, pubrec/2, pubrel/2, pubcomp/2, subscribe/2, subscribe/3, unsubscribe/2]). --behaviour(gen_server2). - %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -135,6 +133,10 @@ -define(PUBSUB_TIMEOUT, 60000). +-define(LOG(Level, Format, Args, State), + lager:Level([{client, State#session.client_id}], + "Session(~s): " ++ Format, [State#session.client_id | Args])). + %%------------------------------------------------------------------------------ %% @doc Start a session. %% @end @@ -151,6 +153,13 @@ start_link(CleanSess, ClientId, ClientPid) -> resume(SessPid, ClientId, ClientPid) -> gen_server2:cast(SessPid, {resume, ClientId, ClientPid}). +%%------------------------------------------------------------------------------ +%% @doc Session Info. +%% @end +%%------------------------------------------------------------------------------ +info(SessPid) -> + gen_server2:call(SessPid, info). + %%------------------------------------------------------------------------------ %% @doc Destroy a session. %% @end @@ -171,7 +180,7 @@ subscribe(SessPid, TopicTable) -> subscribe(SessPid, PacketId, TopicTable) -> From = self(), AckFun = fun(GrantedQos) -> - From ! {suback, PacketId, GrantedQos} + From ! {suback, PacketId, GrantedQos} end, gen_server2:cast(SessPid, {subscribe, TopicTable, AckFun}). @@ -246,12 +255,15 @@ init([CleanSess, ClientId, ClientPid]) -> expired_after = emqttd_opts:g(expired_after, SessEnv) * 3600, collect_interval = emqttd_opts:g(collect_interval, SessEnv, 0), timestamp = os:timestamp()}, - emqttd_sm:register_session(CleanSess, ClientId, info(Session)), + emqttd_sm:register_session(CleanSess, ClientId, sess_info(Session)), %% start statistics {ok, start_collector(Session), hibernate}. prioritise_call(Msg, _From, _Len, _State) -> - case Msg of _ -> 0 end. + case Msg of + info -> 10; + _ -> 0 + end. prioritise_cast(Msg, _Len, _State) -> case Msg of @@ -269,13 +281,16 @@ prioritise_cast(Msg, _Len, _State) -> prioritise_info(Msg, _Len, _State) -> case Msg of {'EXIT', _, _} -> 10; - session_expired -> 10; + expired -> 10; {timeout, _, _} -> 5; collect_info -> 2; {dispatch, _} -> 1; _ -> 0 end. +handle_call(info, _From, State) -> + {reply, sess_info(State), State}; + handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PktId}}, _From, Session = #session{client_id = ClientId, awaiting_rel = AwaitingRel, @@ -292,11 +307,11 @@ handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PktId}}, _From, end; handle_call(Req, _From, State) -> - lager:error("Unexpected Request: ~p", [Req]), - {reply, ok, State}. + ?LOG(critical, "Unexpected Request: ~p", [Req], State), + {reply, {error, unsupported_req}, State}. -handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{ - client_id = ClientId, subscriptions = Subscriptions}) -> +handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{client_id = ClientId, + subscriptions = Subscriptions}) -> TopicTable = emqttd_broker:foldl_hooks('client.subscribe', [ClientId], TopicTable0), @@ -312,18 +327,16 @@ handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{ emqttd_broker:foreach_hooks('client.subscribe.after', [ClientId, TopicTable]), - lager:info([{client, ClientId}], "Session(~s): subscribe ~p, Granted QoS: ~p", - [ClientId, TopicTable, GrantedQos]), + ?LOG(info, "Subscribe ~p, Granted QoS: ~p", [TopicTable, GrantedQos], Session), Subscriptions1 = lists:foldl(fun({Topic, Qos}, Acc) -> case lists:keyfind(Topic, 1, Acc) of {Topic, Qos} -> - lager:warning([{client, ClientId}], "Session(~s): " - "resubscribe ~s, qos = ~w", [ClientId, Topic, Qos]), Acc; + ?LOG(warning, "resubscribe ~s, qos = ~w", [Topic, Qos], Session), + Acc; {Topic, OldQos} -> - lager:warning([{client, ClientId}], "Session(~s): " - "resubscribe ~s, old qos=~w, new qos=~w", [ClientId, Topic, OldQos, Qos]), + ?LOG(warning, "resubscribe ~s, old qos=~w, new qos=~w", [Topic, OldQos, Qos], Session), lists:keyreplace(Topic, 1, Acc, {Topic, Qos}); false -> %%TODO: the design is ugly, rewrite later...:( @@ -353,41 +366,31 @@ handle_cast({unsubscribe, Topics0}, Session = #session{client_id = ClientId, {Topic, _Qos} -> lists:keydelete(Topic, 1, Acc); false -> - lager:warning([{client, ClientId}], "Session(~s) not subscribe ~s", [ClientId, Topic]), Acc + Acc end end, Subscriptions, Topics), noreply(Session#session{subscriptions = Subscriptions1}); handle_cast({destroy, ClientId}, Session = #session{client_id = ClientId}) -> - lager:warning([{client, ClientId}], "Session(~s) destroyed", [ClientId]), + ?LOG(warning, "destroyed", [], Session), {stop, {shutdown, destroy}, Session}; -handle_cast({resume, ClientId, ClientPid}, Session) -> +handle_cast({resume, ClientId, ClientPid}, Session = #session{client_id = ClientId, + client_pid = OldClientPid, + inflight_queue = InflightQ, + awaiting_ack = AwaitingAck, + awaiting_comp = AwaitingComp, + expired_timer = ETimer} = Session) -> - #session{client_id = ClientId, - client_pid = OldClientPid, - inflight_queue = InflightQ, - awaiting_ack = AwaitingAck, - awaiting_comp = AwaitingComp, - expired_timer = ETimer} = Session, + ?LOG(info, "resumed by ~p", [ClientPid], Session), - lager:info([{client, ClientId}], "Session(~s) resumed by ~p", [ClientId, ClientPid]), - - %% cancel expired timer + %% Cancel expired timer cancel_timer(ETimer), - %% Kickout old client - if - OldClientPid == undefined -> - ok; - OldClientPid == ClientPid -> - ok; %% ?? - true -> - lager:error([{client, ClientId}], "Session(~s): ~p kickout ~p", - [ClientId, ClientPid, OldClientPid]), - unlink(OldClientPid), - OldClientPid ! {stop, duplicate_id, ClientPid} + case kick(ClientId, OldClientPid, ClientPid) of + ok -> ?LOG(warning, "~p kickout ~p", [ClientPid, OldClientPid], Session); + ignore -> ok end, true = link(ClientPid), @@ -416,19 +419,18 @@ handle_cast({resume, ClientId, ClientPid}, Session) -> noreply(dequeue(Session2)); %% PUBACK -handle_cast({puback, PktId}, Session = #session{client_id = ClientId, awaiting_ack = AwaitingAck}) -> +handle_cast({puback, PktId}, Session = #session{awaiting_ack = AwaitingAck}) -> case maps:find(PktId, AwaitingAck) of {ok, TRef} -> cancel_timer(TRef), noreply(dequeue(acked(PktId, Session))); error -> - lager:error([{client, ClientId}], "Session(~s) cannot find PUBACK ~w", [ClientId, PktId]), + ?LOG(error, "Cannot find PUBACK: ~p", [PktId], Session), noreply(Session) end; %% PUBREC -handle_cast({pubrec, PktId}, Session = #session{client_id = ClientId, - awaiting_ack = AwaitingAck, +handle_cast({pubrec, PktId}, Session = #session{awaiting_ack = AwaitingAck, awaiting_comp = AwaitingComp, await_rel_timeout = Timeout}) -> case maps:find(PktId, AwaitingAck) of @@ -439,37 +441,36 @@ handle_cast({pubrec, PktId}, Session = #session{client_id = ClientId, Session1 = acked(PktId, Session#session{awaiting_comp = AwaitingComp1}), noreply(dequeue(Session1)); error -> - lager:error([{client, ClientId}], "Session(~s) cannot find PUBREC ~w", [ClientId, PktId]), + ?LOG(error, "Cannot find PUBREC: ~p", [PktId], Session), noreply(Session) end; %% PUBREL -handle_cast({pubrel, PktId}, Session = #session{client_id = ClientId, - awaiting_rel = AwaitingRel}) -> +handle_cast({pubrel, PktId}, Session = #session{awaiting_rel = AwaitingRel}) -> case maps:find(PktId, AwaitingRel) of {ok, {Msg, TRef}} -> cancel_timer(TRef), emqttd_pubsub:publish(Msg), noreply(Session#session{awaiting_rel = maps:remove(PktId, AwaitingRel)}); error -> - lager:error([{client, ClientId}], "Session(~s) cannot find PUBREL ~w", [ClientId, PktId]), + ?LOG(error, "Cannot find PUBREL: ~p", [PktId], Session), noreply(Session) end; %% PUBCOMP -handle_cast({pubcomp, PktId}, Session = #session{client_id = ClientId, awaiting_comp = AwaitingComp}) -> +handle_cast({pubcomp, PktId}, Session = #session{awaiting_comp = AwaitingComp}) -> case maps:find(PktId, AwaitingComp) of {ok, TRef} -> cancel_timer(TRef), noreply(Session#session{awaiting_comp = maps:remove(PktId, AwaitingComp)}); error -> - lager:error("Session(~s) cannot find PUBCOMP ~w", [ClientId, PktId]), + ?LOG(error, "Cannot find PUBCOMP: ~p", [PktId], Session), noreply(Session) end; handle_cast(Msg, State) -> - lager:error("Unexpected Msg: ~p, State: ~p", [Msg, State]), - {noreply, State}. + ?LOG(critical, "Unexpected Msg: ~p", [Msg], State), + noreply(State). %% Queue messages when client is offline handle_info({dispatch, Msg}, Session = #session{client_pid = undefined, @@ -483,14 +484,15 @@ handle_info({dispatch, Msg = #mqtt_message{qos = ?QOS_0}}, ClientPid ! {deliver, Msg}, noreply(Session); -handle_info({dispatch, Msg = #mqtt_message{qos = QoS}}, Session = #session{message_queue = MsgQ}) +handle_info({dispatch, Msg = #mqtt_message{qos = QoS}}, + Session = #session{message_queue = MsgQ}) when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> case check_inflight(Session) of - true -> - {noreply, deliver(Msg, Session)}; + true -> + noreply(deliver(Msg, Session)); false -> - {noreply, Session#session{message_queue = emqttd_mqueue:in(Msg, MsgQ)}} + noreply(Session#session{message_queue = emqttd_mqueue:in(Msg, MsgQ)}) end; handle_info({timeout, awaiting_ack, PktId}, Session = #session{client_pid = undefined, @@ -498,81 +500,70 @@ handle_info({timeout, awaiting_ack, PktId}, Session = #session{client_pid = unde %% just remove awaiting noreply(Session#session{awaiting_ack = maps:remove(PktId, AwaitingAck)}); -handle_info({timeout, awaiting_ack, PktId}, Session = #session{client_id = ClientId, - inflight_queue = InflightQ, +handle_info({timeout, awaiting_ack, PktId}, Session = #session{inflight_queue = InflightQ, awaiting_ack = AwaitingAck}) -> - lager:info("Awaiting Ack Timeout: ~p:", [PktId]), case maps:find(PktId, AwaitingAck) of {ok, _TRef} -> case lists:keyfind(PktId, 1, InflightQ) of {_, Msg} -> noreply(redeliver(Msg, Session)); false -> - lager:error([{client, ClientId}], "Session(~s):" - "Awaiting timeout but Cannot find PktId :~p", [ClientId, PktId]), + ?LOG(error, "AwaitingAck timeout but Cannot find PktId: ~p", [PktId], Session), noreply(dequeue(Session)) end; error -> - lager:error([{client, ClientId}], "Session(~s):" - "Cannot find Awaiting Ack:~p", [ClientId, PktId]), + ?LOG(error, "Cannot find AwaitingAck: ~p", [PktId], Session), noreply(Session) end; -handle_info({timeout, awaiting_rel, PktId}, Session = #session{client_id = ClientId, - awaiting_rel = AwaitingRel}) -> +handle_info({timeout, awaiting_rel, PktId}, Session = #session{awaiting_rel = AwaitingRel}) -> case maps:find(PktId, AwaitingRel) of - {ok, {Msg, _TRef}} -> - lager:error([{client, ClientId}], "Session(~s) AwaitingRel Timout!~n" - "Drop Message:~p", [ClientId, Msg]), + {ok, {_Msg, _TRef}} -> + ?LOG(error, "AwaitingRel Timout: ~p, Drop Message!", [PktId], Session), noreply(Session#session{awaiting_rel = maps:remove(PktId, AwaitingRel)}); error -> - lager:error([{client, ClientId}], "Session(~s) cannot find AwaitingRel ~w", [ClientId, PktId]), - {noreply, Session, hibernate} + ?LOG(error, "Cannot find AwaitingRel: ~p", [PktId], Session), + noreply(Session) end; -handle_info({timeout, awaiting_comp, PktId}, Session = #session{client_id = ClientId, - awaiting_comp = Awaiting}) -> +handle_info({timeout, awaiting_comp, PktId}, Session = #session{awaiting_comp = Awaiting}) -> case maps:find(PktId, Awaiting) of {ok, _TRef} -> - lager:error([{client, ClientId}], "Session(~s) " - "Awaiting PUBCOMP Timout: PktId=~p!", [ClientId, PktId]), + ?LOG(error, "Awaiting PUBCOMP Timout: ~p", [PktId], Session), noreply(Session#session{awaiting_comp = maps:remove(PktId, Awaiting)}); error -> - lager:error([{client, ClientId}], "Session(~s) " - "Cannot find Awaiting PUBCOMP: PktId=~p", [ClientId, PktId]), + ?LOG(error, "Cannot find Awaiting PUBCOMP: ~p", [PktId], Session), noreply(Session) end; handle_info(collect_info, Session = #session{clean_sess = CleanSess, client_id = ClientId}) -> - emqttd_sm:register_session(CleanSess, ClientId, info(Session)), - {noreply, start_collector(Session), hibernate}; + emqttd_sm:register_session(CleanSess, ClientId, sess_info(Session)), + noreply(start_collector(Session)); handle_info({'EXIT', ClientPid, _Reason}, Session = #session{clean_sess = true, client_pid = ClientPid}) -> {stop, normal, Session}; handle_info({'EXIT', ClientPid, Reason}, Session = #session{clean_sess = false, - client_id = ClientId, client_pid = ClientPid, expired_after = Expires}) -> - lager:info("Session(~s): unlink with client ~p: reason=~p", - [ClientId, ClientPid, Reason]), - TRef = timer(Expires, session_expired), + ?LOG(info, "Client ~p EXIT for ~p", [ClientPid, Reason], Session), + TRef = timer(Expires, expired), + erlang:garbage_collect(), %%TODO: ??? noreply(Session#session{client_pid = undefined, expired_timer = TRef}); -handle_info({'EXIT', Pid, Reason}, Session = #session{client_id = ClientId, - client_pid = ClientPid}) -> +handle_info({'EXIT', Pid, Reason}, Session = #session{client_pid = ClientPid}) -> - lager:error("Session(~s): Unexpected EXIT: client_pid=~p, exit_pid=~p, reason=~p", - [ClientId, ClientPid, Pid, Reason]), + ?LOG(error, "Unexpected EXIT: client_pid=~p, exit_pid=~p, reason=~p", + [ClientPid, Pid, Reason], Session), noreply(Session); -handle_info(session_expired, Session = #session{client_id = ClientId}) -> - lager:error("Session(~s) expired, shutdown now.", [ClientId]), +handle_info(expired, Session) -> + ?LOG(info, "expired, shutdown now.", [], Session), {stop, {shutdown, expired}, Session}; -handle_info(Info, Session = #session{client_id = ClientId}) -> - lager:error("Session(~s) unexpected info: ~p", [ClientId, Info]), +handle_info(Info, Session) -> + ?LOG(critical, "Unexpected info: ~p", [Info], Session), {noreply, Session}. terminate(_Reason, #session{clean_sess = CleanSess, client_id = ClientId}) -> @@ -585,6 +576,17 @@ code_change(_OldVsn, Session, _Extra) -> %%% Internal functions %%%============================================================================= +%%------------------------------------------------------------------------------ +%% Kick old client out +%%------------------------------------------------------------------------------ +kick(_ClientId, undefined, _Pid) -> + ignore; +kick(_ClientId, Pid, Pid) -> + ignore; +kick(ClientId, OldPid, Pid) -> + unlink(OldPid), + OldPid ! {shutdown, conflict, {ClientId, Pid}}. + %%------------------------------------------------------------------------------ %% Check inflight and awaiting_rel %%------------------------------------------------------------------------------ @@ -658,7 +660,7 @@ acked(PktId, Session = #session{client_id = ClientId, {_, Msg} -> emqttd_broker:foreach_hooks('message.acked', [ClientId, Msg]); false -> - lager:error("Session(~s): Cannot find acked message: ~p", [PktId]) + ?LOG(error, "Cannot find acked pktid: ~p", [PktId], Session) end, Session#session{awaiting_ack = maps:remove(PktId, Awaiting), inflight_queue = lists:keydelete(PktId, 1, InflightQ)}. @@ -687,15 +689,15 @@ start_collector(Session = #session{collect_interval = Interval}) -> TRef = erlang:send_after(timer:seconds(Interval), self(), collect_info), Session#session{collect_timer = TRef}. -info(#session{clean_sess = CleanSess, - subscriptions = Subscriptions, - inflight_queue = InflightQueue, - max_inflight = MaxInflight, - message_queue = MessageQueue, - awaiting_rel = AwaitingRel, - awaiting_ack = AwaitingAck, - awaiting_comp = AwaitingComp, - timestamp = CreatedAt}) -> +sess_info(#session{clean_sess = CleanSess, + subscriptions = Subscriptions, + inflight_queue = InflightQueue, + max_inflight = MaxInflight, + message_queue = MessageQueue, + awaiting_rel = AwaitingRel, + awaiting_ack = AwaitingAck, + awaiting_comp = AwaitingComp, + timestamp = CreatedAt}) -> Stats = emqttd_mqueue:stats(MessageQueue), [{clean_sess, CleanSess}, {subscriptions, Subscriptions}, diff --git a/src/emqttd_sm.erl b/src/emqttd_sm.erl index 719ac0ca1..d62e85c86 100644 --- a/src/emqttd_sm.erl +++ b/src/emqttd_sm.erl @@ -57,7 +57,10 @@ -define(SM_POOL, ?MODULE). --define(SESSION_TIMEOUT, 60000). +-define(CALL_TIMEOUT, 60000). + +-define(LOG(Level, Format, Args, Session), + lager:Level("SM(~s): " ++ Format, [Session#mqtt_session.client_id | Args])). %%%============================================================================= %%% Mnesia callbacks @@ -113,7 +116,7 @@ start_session(CleanSess, ClientId) -> lookup_session(ClientId) -> case mnesia:dirty_read(session, ClientId) of [Session] -> Session; - [] -> undefined + [] -> undefined end. %%------------------------------------------------------------------------------ @@ -137,13 +140,11 @@ register_session(CleanSess, ClientId, Info) -> unregister_session(CleanSess, ClientId) -> ets:delete(sesstab(CleanSess), {ClientId, self()}). -sesstab(true) -> - mqtt_transient_session; -sesstab(false) -> - mqtt_persistent_session. +sesstab(true) -> mqtt_transient_session; +sesstab(false) -> mqtt_persistent_session. call(SM, Req) -> - gen_server2:call(SM, Req, ?SESSION_TIMEOUT). %%infinity). + gen_server2:call(SM, Req, ?CALL_TIMEOUT). %%infinity). %%%============================================================================= %%% gen_server callbacks @@ -223,8 +224,8 @@ create_session(CleanSess, ClientId, ClientPid) -> case insert_session(Session) of {aborted, {conflict, ConflictPid}} -> %% Conflict with othe node? - lager:error("Session(~s): Conflict with ~p!", [ClientId, ConflictPid]), - {error, conflict}; + lager:error("SM(~s): Conflict with ~p", [ClientId, ConflictPid]), + {error, mnesia_conflict}; {atomic, ok} -> erlang:monitor(process, SessPid), {ok, SessPid} @@ -245,8 +246,8 @@ insert_session(Session = #mqtt_session{client_id = ClientId}) -> end). %% Local node -resume_session(#mqtt_session{client_id = ClientId, - sess_pid = SessPid}, ClientPid) +resume_session(Session = #mqtt_session{client_id = ClientId, + sess_pid = SessPid}, ClientPid) when node(SessPid) =:= node() -> case is_process_alive(SessPid) of @@ -254,7 +255,7 @@ resume_session(#mqtt_session{client_id = ClientId, emqttd_session:resume(SessPid, ClientId, ClientPid), {ok, SessPid}; false -> - lager:error("Session(~s): Cannot resume ~p, it seems already dead!", [ClientId, SessPid]), + ?LOG(error, "Cannot resume ~p which seems already dead!", [SessPid], Session), {error, session_died} end; @@ -265,12 +266,11 @@ resume_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid} ok -> {ok, SessPid}; {badrpc, nodedown} -> - lager:error("Session(~s): Died for node ~s down!", [ClientId, Node]), + ?LOG(error, "Session died for node '~s' down", [Node], Session), remove_session(Session), {error, session_nodedown}; {badrpc, Reason} -> - lager:error("Session(~s): Failed to resume from node ~s for ~p", - [ClientId, Node, Reason]), + ?LOG(error, "Failed to resume from node ~s for ~p", [Node, Reason], Session), {error, Reason} end. @@ -288,11 +288,11 @@ destroy_session(Session = #mqtt_session{client_id = ClientId, ok -> remove_session(Session); {badrpc, nodedown} -> - lager:error("Session(~s): Died for node ~s down!", [ClientId, Node]), + ?LOG(error, "Node '~s' down", [Node], Session), remove_session(Session); {badrpc, Reason} -> - lager:error("Session(~s): Failed to destory ~p on remote node ~p for ~s", - [ClientId, SessPid, Node, Reason]), + ?LOG(error, "Failed to destory ~p on remote node ~p for ~s", + [SessPid, Node, Reason], Session), {error, Reason} end. From b9962d697cc014626ee4b1376840e91c57cca418 Mon Sep 17 00:00:00 2001 From: Feng Date: Fri, 30 Oct 2015 21:23:17 +0800 Subject: [PATCH 21/37] 0.13.0 --- CHANGELOG.md | 10 ++++++++++ src/emqttd.app.src | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04ed16428..a7bd8d018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ emqttd ChangeLog ================== +0.13.0-alpha (2015-11-02) +------------------------- + +eSockd 3.0 + +MochiWeb 4.0 + +...... + + 0.12.3-beta (2015-10-22) ------------------------- diff --git a/src/emqttd.app.src b/src/emqttd.app.src index 7591c61d5..8ebfbe721 100644 --- a/src/emqttd.app.src +++ b/src/emqttd.app.src @@ -1,7 +1,7 @@ {application, emqttd, [ {id, "emqttd"}, - {vsn, "0.12.4"}, + {vsn, "0.13.0"}, {description, "Erlang MQTT Broker"}, {modules, []}, {registered, []}, From f9027ed1f43262363891fa94a493f2cfcc1f980b Mon Sep 17 00:00:00 2001 From: Feng Date: Sat, 31 Oct 2015 15:07:50 +0800 Subject: [PATCH 22/37] fix issue #367 - Fix the order of emqttd_app:start_server/1 --- src/emqttd_app.erl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/emqttd_app.erl b/src/emqttd_app.erl index fa7799904..53adfa582 100644 --- a/src/emqttd_app.erl +++ b/src/emqttd_app.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_app). -author("Feng Lee "). @@ -73,14 +72,14 @@ start_listeners() -> start_servers(Sup) -> Servers = [{"emqttd ctl", emqttd_ctl}, {"emqttd trace", emqttd_trace}, + {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}}, + {"emqttd stats", emqttd_stats}, + {"emqttd metrics", emqttd_metrics}, {"emqttd retained", emqttd_retained}, {"emqttd pooler", {supervisor, emqttd_pooler_sup}}, {"emqttd client manager", {supervisor, emqttd_cm_sup}}, {"emqttd session manager", {supervisor, emqttd_sm_sup}}, {"emqttd session supervisor", {supervisor, emqttd_session_sup}}, - {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}}, - {"emqttd stats", emqttd_stats}, - {"emqttd metrics", emqttd_metrics}, {"emqttd broker", emqttd_broker}, {"emqttd alarm", emqttd_alarm}, {"emqttd mode supervisor", emqttd_mod_sup}, From 3f41a6c241a18724cb1d3f7a980b0bd2e8dcc389 Mon Sep 17 00:00:00 2001 From: Feng Date: Sat, 31 Oct 2015 15:08:25 +0800 Subject: [PATCH 23/37] peername --- src/emqttd_client.erl | 15 ++++++++------- src/emqttd_packet.erl | 23 ++++++++++++++--------- src/emqttd_protocol.erl | 12 ++++-------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 125ac40df..40b8a59cd 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -47,17 +47,16 @@ code_change/3, terminate/2]). %% Client State --record(client_state, {connection, peername, peerhost, peerport, - await_recv, conn_state, rate_limit, - parser_fun, proto_state, packet_opts, - keepalive}). +-record(client_state, {connection, connname, peername, peerhost, peerport, + await_recv, conn_state, rate_limit, parser_fun, + proto_state, packet_opts, keepalive}). -define(INFO_KEYS, [peername, peerhost, peerport, await_recv, conn_state]). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). -define(LOG(Level, Format, Args, State), - lager:Level("Client(~s): " ++ Format, [State#client_state.peername | Args])). + lager:Level("Client(~s): " ++ Format, [State#client_state.connname | Args])). start_link(Connection, MqttEnv) -> {ok, proc_lib:spawn_link(?MODULE, init, [[Connection, MqttEnv]])}. @@ -81,8 +80,8 @@ init([Connection0, MqttEnv]) -> {ok, Connection} = Connection0:wait(), {PeerHost, PeerPort, PeerName} = case Connection:peername() of - {ok, {Host, Port}} -> - {Host, Port, esockd_net:format({Host, Port})}; + {ok, Peer = {Host, Port}} -> + {Host, Port, Peer}; {error, enotconn} -> Connection:fast_close(), exit(normal); @@ -90,6 +89,7 @@ init([Connection0, MqttEnv]) -> Connection:fast_close(), exit({shutdown, Reason}) end, + ConnName = esockd_net:format(PeerName), SendFun = fun(Data) -> try Connection:async_send(Data) of true -> ok @@ -102,6 +102,7 @@ init([Connection0, MqttEnv]) -> ProtoState = emqttd_protocol:init(PeerName, SendFun, PktOpts), RateLimit = proplists:get_value(rate_limit, Connection:opts()), State = run_socket(#client_state{connection = Connection, + connname = ConnName, peername = PeerName, peerhost = PeerHost, peerport = PeerPort, diff --git a/src/emqttd_packet.erl b/src/emqttd_packet.erl index 383f8df7d..0bc5190a8 100644 --- a/src/emqttd_packet.erl +++ b/src/emqttd_packet.erl @@ -74,13 +74,15 @@ connack_name(?CONNACK_AUTH) -> 'CONNACK_AUTH'. format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) -> format_header(Header, format_variable(Variable, Payload)). -format_header(#mqtt_packet_header{type = Type, dup = Dup, qos = QoS, retain = Retain}, S) -> - S1 = - if - S == undefined -> <<>>; - true -> [", ", S] - end, - io_lib:format("~s(Qos=~p, Retain=~s, Dup=~s~s)", [type_name(Type), QoS, Retain, Dup, S1]). +format_header(#mqtt_packet_header{type = Type, + dup = Dup, + qos = QoS, + retain = Retain}, S) -> + S1 = if + S == undefined -> <<>>; + true -> [", ", S] + end, + io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). format_variable(undefined, _) -> undefined; @@ -105,8 +107,8 @@ format_variable(#mqtt_packet_connect{ Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanSess=~s, KeepAlive=~p, Username=~s, Password=~s", Args = [ClientId, ProtoName, ProtoVer, CleanSess, KeepAlive, Username, format_password(Password)], {Format1, Args1} = if - WillFlag -> { Format ++ ", Will(Qos=~p, Retain=~s, Topic=~s, Msg=~s)", - Args ++ [ WillQoS, WillRetain, WillTopic, WillMsg ] }; + WillFlag -> { Format ++ ", Will(Q~p, R~p, Topic=~s, Msg=~s)", + Args ++ [WillQoS, i(WillRetain), WillTopic, WillMsg] }; true -> {Format, Args} end, io_lib:format(Format1, Args1); @@ -145,3 +147,6 @@ format_variable(undefined) -> undefined. format_password(undefined) -> undefined; format_password(_Password) -> '******'. +i(true) -> 1; +i(false) -> 0; +i(I) when is_integer(I) -> I. diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index a2311ec7d..d0af0d420 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -56,7 +56,7 @@ -define(LOG(Level, Format, Args, State), lager:Level([{client, State#proto_state.client_id}], "Client(~s@~s): " ++ Format, - [State#proto_state.client_id, State#proto_state.peername | Args])). + [State#proto_state.client_id, esockd_net:format(State#proto_state.peername) | Args])). %%------------------------------------------------------------------------------ %% @doc Init protocol @@ -269,14 +269,10 @@ send(Packet, State = #proto_state{sendfun = SendFun}) {ok, State}. trace(recv, Packet, ProtoState) -> - trace2("RECV <-", Packet, ProtoState); + ?LOG(info, "RECV ~s", [emqttd_packet:format(Packet)], ProtoState); trace(send, Packet, ProtoState) -> - trace2("SEND ->", Packet, ProtoState). - -trace2(Tag, Packet, #proto_state{peername = Peername, client_id = ClientId}) -> - lager:info([{client, ClientId}], "Client(~s@~s): ~s ~s", - [ClientId, Peername, Tag, emqttd_packet:format(Packet)]). + ?LOG(info, "SEND ~s", [emqttd_packet:format(Packet)], ProtoState). %% @doc redeliver PUBREL PacketId redeliver({?PUBREL, PacketId}, State) -> @@ -289,7 +285,7 @@ shutdown(confict, #proto_state{client_id = ClientId}) -> emqttd_cm:unregister(ClientId); shutdown(Error, State = #proto_state{client_id = ClientId, will_msg = WillMsg}) -> - ?LOG(info, "shutdown for ~p", [Error], State), + ?LOG(info, "Shutdown for ~p", [Error], State), send_willmsg(ClientId, WillMsg), emqttd_broker:foreach_hooks('client.disconnected', [Error, ClientId]), emqttd_cm:unregister(ClientId). From cf95c5e1b9969d881aa09e65f14a1aeb6d373e4f Mon Sep 17 00:00:00 2001 From: Feng Date: Sat, 31 Oct 2015 18:43:04 +0800 Subject: [PATCH 24/37] 0.13 refactor --- src/emqttd_client.erl | 2 +- src/emqttd_message.erl | 19 ++--- src/emqttd_protocol.erl | 2 +- src/emqttd_retained.erl | 2 +- src/emqttd_ws_client.erl | 165 ++++++++++++++++++++++----------------- 5 files changed, 104 insertions(+), 86 deletions(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 40b8a59cd..f471d311b 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -177,7 +177,7 @@ handle_info(activate_sock, State) -> handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> Size = size(Data), - ?LOG(debug, "RECV <- ~p", [Data], State), + ?LOG(debug, "RECV ~p", [Data], State), emqttd_metrics:inc('bytes/received', Size), received(Data, rate_limit(Size, State#client_state{await_recv = false})); diff --git a/src/emqttd_message.erl b/src/emqttd_message.erl index d8e55e202..9d340b2b3 100644 --- a/src/emqttd_message.erl +++ b/src/emqttd_message.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_message). -author("Feng Lee "). @@ -170,14 +169,12 @@ unset_flag(Flag, Msg) when Flag =:= dup orelse Flag =:= retain -> Msg. %% @doc Format MQTT Message %% @end %%------------------------------------------------------------------------------ -format(#mqtt_message{msgid=MsgId, - pktid = PktId, - from = From, - qos=Qos, - retain=Retain, - dup=Dup, - topic=Topic}) -> - io_lib:format("Message(MsgId=~p, PktId=~p, from=~s, " - "Qos=~p, Retain=~s, Dup=~s, Topic=~s)", - [MsgId, PktId, From, Qos, Retain, Dup, Topic]). +format(#mqtt_message{msgid = MsgId, pktid = PktId, from = From, + qos = Qos, retain = Retain, dup = Dup, topic =Topic}) -> + io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s, Topic=~s)", + [i(Qos), i(Retain), i(Dup), MsgId, PktId, From, Topic]). + +i(true) -> 1; +i(false) -> 0; +i(I) when is_integer(I) -> I. diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index d0af0d420..954332afe 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -263,7 +263,7 @@ send(Packet, State = #proto_state{sendfun = SendFun}) trace(send, Packet, State), emqttd_metrics:sent(Packet), Data = emqttd_serialiser:serialise(Packet), - ?LOG(debug, "SENT -> ~p", [Data], State), + ?LOG(debug, "SEND ~p", [Data], State), emqttd_metrics:inc('bytes/sent', size(Data)), SendFun(Data), {ok, State}. diff --git a/src/emqttd_retained.erl b/src/emqttd_retained.erl index 8e797255b..685006200 100644 --- a/src/emqttd_retained.erl +++ b/src/emqttd_retained.erl @@ -98,7 +98,7 @@ retain(Msg = #mqtt_message{topic = Topic, retain = true, payload = Payload}) -> case {TabSize < limit(table), size(Payload) < limit(payload)} of {true, true} -> Retained = #mqtt_retained{topic = Topic, message = Msg}, - lager:debug("Retained ~s", [emqttd_message:format(Msg)]), + lager:debug("RETAIN ~s", [emqttd_message:format(Msg)]), mnesia:async_dirty(fun mnesia:write/3, [retained, Retained, write]), emqttd_metrics:set('messages/retained', mnesia:table_info(retained, size)); {false, _}-> diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index b4f82c043..1d02ab46c 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_ws_client). -author("Feng Lee "). @@ -46,10 +45,13 @@ terminate/2, code_change/3]). %% WebSocket Loop State --record(wsocket_state, {request, client_pid, packet_opts, parser}). +-record(wsocket_state, {request, client_pid, packet_opts, parser_fun}). -%% Client State --record(client_state, {ws_pid, request, proto_state, keepalive}). +%% WebSocket Client State +-record(wsclient_state, {ws_pid, request, proto_state, keepalive}). + +-define(WSLOG(Level, Format, Args, Req), + lager:Level("WsClient(~s): " ++ Format, [Req:get(peer) | Args])). %%------------------------------------------------------------------------------ %% @doc Start WebSocket client. @@ -57,12 +59,14 @@ %%------------------------------------------------------------------------------ start_link(Req) -> PktOpts = emqttd:env(mqtt, packet), + ParserFun = emqttd_parser:new(PktOpts), {ReentryWs, ReplyChannel} = upgrade(Req), - {ok, ClientPid} = gen_server:start_link(?MODULE, [self(), Req, ReplyChannel, PktOpts], []), - ReentryWs(#wsocket_state{request = Req, - client_pid = ClientPid, - packet_opts = PktOpts, - parser = emqttd_parser:new(PktOpts)}). + Params = [self(), Req, ReplyChannel, PktOpts], + {ok, ClientPid} = gen_server:start_link(?MODULE, Params, []), + ReentryWs(#wsocket_state{request = Req, + client_pid = ClientPid, + packet_opts = PktOpts, + parser_fun = ParserFun}). session(CPid) -> gen_server:call(CPid, session, infinity). @@ -97,22 +101,25 @@ ws_loop([<<>>], State, _ReplyChannel) -> State; ws_loop(Data, State = #wsocket_state{request = Req, client_pid = ClientPid, - parser = Parser}, ReplyChannel) -> - Peer = Req:get(peer), - lager:debug("RECV from ~s(WebSocket): ~p", [Peer, Data]), - case Parser(iolist_to_binary(Data)) of + parser_fun = ParserFun}, ReplyChannel) -> + ?WSLOG(debug, "RECV ~p", [Data], Req), + case catch ParserFun(iolist_to_binary(Data)) of {more, NewParser} -> - State#wsocket_state{parser = NewParser}; + State#wsocket_state{parser_fun = NewParser}; {ok, Packet, Rest} -> gen_server:cast(ClientPid, {received, Packet}), ws_loop(Rest, reset_parser(State), ReplyChannel); {error, Error} -> - lager:error("MQTT(WebSocket) frame error ~p for connection ~s", [Error, Peer]), - exit({shutdown, Error}) + ?WSLOG(error, "Frame error: ~p", [Error], Req), + exit({shutdown, Error}); + {'EXIT', Reason} -> + ?WSLOG(error, "Frame error: ~p", [Reason], Req), + ?WSLOG(error, "Error data: ~p", [Data], Req), + exit({shutdown, parser_error}) end. reset_parser(State = #wsocket_state{packet_opts = PktOpts}) -> - State#wsocket_state{parser = emqttd_parser:new(PktOpts)}. + State#wsocket_state{parser_fun = emqttd_parser:new(PktOpts)}. %%%============================================================================= %%% gen_server callbacks @@ -125,98 +132,105 @@ init([WsPid, Req, ReplyChannel, PktOpts]) -> Headers = mochiweb_request:get(headers, Req), HeadersList = mochiweb_headers:to_list(Headers), ProtoState = emqttd_protocol:init(Peername, SendFun, - [{ws_initial_headers, HeadersList}|PktOpts]), - {ok, #client_state{ws_pid = WsPid, request = Req, proto_state = ProtoState}}. + [{ws_initial_headers, HeadersList} | PktOpts]), + {ok, #wsclient_state{ws_pid = WsPid, request = Req, proto_state = ProtoState}}. -handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> +handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) -> {reply, emqttd_protocol:session(ProtoState), State}; -handle_call(info, _From, State = #client_state{request = Req, - proto_state = ProtoState}) -> - {reply, [{websocket, true}, {peer, Req:get(peer)} - | emqttd_protocol:info(ProtoState)], State}; +handle_call(info, _From, State = #wsclient_state{request = Req, + proto_state = ProtoState}) -> + ProtoInfo = emqttd_protocol:info(ProtoState), + {reply, [{websocket, true}, {peer, Req:get(peer)}| ProtoInfo], State}; handle_call(kick, _From, State) -> {stop, {shutdown, kick}, ok, State}; -handle_call(_Req, _From, State) -> - {reply, error, State}. +handle_call(Req, _From, State = #wsclient_state{request = HttpReq}) -> + ?WSLOG(critical, "Unexpected request: ~p", [Req], HttpReq), + {reply, {error, unsupported_request}, State}. handle_cast({subscribe, TopicTable}, State) -> - with_session(fun(SessPid) -> emqttd_session:subscribe(SessPid, TopicTable) end, State); + with_session(fun(SessPid) -> + emqttd_session:subscribe(SessPid, TopicTable) + end, State); handle_cast({unsubscribe, Topics}, State) -> - with_session(fun(SessPid) -> emqttd_session:unsubscribe(SessPid, Topics) end, State); + with_session(fun(SessPid) -> + emqttd_session:unsubscribe(SessPid, Topics) + end, State); -handle_cast({received, Packet}, State = #client_state{proto_state = ProtoState}) -> +handle_cast({received, Packet}, State = #wsclient_state{request = Req, + proto_state = ProtoState}) -> case emqttd_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - noreply(State#client_state{proto_state = ProtoState1}); + noreply(State#wsclient_state{proto_state = ProtoState1}); {error, Error} -> - lager:error("MQTT protocol error ~p", [Error]), - stop({shutdown, Error}, State); + ?WSLOG(error, "Protocol error - ~p", [Error], Req), + shutdown(Error, State); {error, Error, ProtoState1} -> - stop({shutdown, Error}, State#client_state{proto_state = ProtoState1}); + shutdown(Error, State#wsclient_state{proto_state = ProtoState1}); {stop, Reason, ProtoState1} -> - stop(Reason, State#client_state{proto_state = ProtoState1}) + stop(Reason, State#wsclient_state{proto_state = ProtoState1}) end; -handle_cast(_Msg, State) -> +handle_cast(Msg, State = #wsclient_state{request = Req}) -> + ?WSLOG(critical, "Unexpected msg: ~p", [Msg], Req), {noreply, State}. -%% Asynchronous SUBACK -handle_info({suback, PacketId, GrantedQos}, State = #client_state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:send(?SUBACK_PACKET(PacketId, GrantedQos), ProtoState), - noreply(State#client_state{proto_state = ProtoState1}); +handle_info({suback, PacketId, GrantedQos}, State) -> + with_proto_state(fun(ProtoState) -> + Packet = ?SUBACK_PACKET(PacketId, GrantedQos), + emqttd_protocol:send(Packet, ProtoState) + end, State); -handle_info({deliver, Message}, State = #client_state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:send(Message, ProtoState), - noreply(State#client_state{proto_state = ProtoState1}); +handle_info({deliver, Message}, State) -> + with_proto_state(fun(ProtoState) -> + emqttd_protocol:send(Message, ProtoState) + end, State); -handle_info({redeliver, {?PUBREL, PacketId}}, State = #client_state{proto_state = ProtoState}) -> - {ok, ProtoState1} = emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState), - noreply(State#client_state{proto_state = ProtoState1}); +handle_info({redeliver, {?PUBREL, PacketId}}, State) -> + with_proto_state(fun(ProtoState) -> + emqttd_protocol:redeliver({?PUBREL, PacketId}, ProtoState) + end, State); -handle_info({stop, duplicate_id, _NewPid}, State = #client_state{proto_state = ProtoState}) -> - lager:error("Shutdown for duplicate clientid: ~s", [emqttd_protocol:clientid(ProtoState)]), - stop({shutdown, duplicate_id}, State); +handle_info({shutdown, conflict, {ClientId, NewPid}}, State = #wsclient_state{request = Req}) -> + ?WSLOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], Req), + shutdown(confict, State); -handle_info({keepalive, start, TimeoutSec}, State = #client_state{request = Req}) -> - lager:debug("Client(WebSocket) ~s: Start KeepAlive with ~p seconds", [Req:get(peer), TimeoutSec]), - Socket = Req:get(socket), +handle_info({keepalive, start, Interval}, State = #wsclient_state{request = Req}) -> + ?WSLOG(debug, "Keepalive at the interval of ~p", [Interval], Req), + Conn = Req:get(connection), StatFun = fun() -> - case esockd_transport:getstat(Socket, [recv_oct]) of + case Conn:getstat([recv_oct]) of {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; {error, Error} -> {error, Error} end end, - KeepAlive = emqttd_keepalive:start(StatFun, TimeoutSec, {keepalive, check}), - noreply(State#client_state{keepalive = KeepAlive}); + KeepAlive = emqttd_keepalive:start(StatFun, Interval, {keepalive, check}), + noreply(State#wsclient_state{keepalive = KeepAlive}); -handle_info({keepalive, check}, State = #client_state{request = Req, keepalive = KeepAlive}) -> +handle_info({keepalive, check}, State = #wsclient_state{request = Req, + keepalive = KeepAlive}) -> case emqttd_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> - noreply(State#client_state{keepalive = KeepAlive1}); + noreply(State#wsclient_state{keepalive = KeepAlive1}); {error, timeout} -> - lager:debug("Client(WebSocket) ~s: Keepalive Timeout!", [Req:get(peer)]), - stop({shutdown, keepalive_timeout}, State#client_state{keepalive = undefined}); + ?WSLOG(debug, "Keepalive Timeout!", [], Req), + shutdown(keepalive_timeout, State); {error, Error} -> - lager:debug("Client(WebSocket) ~s: Keepalive Error: ~p", [Req:get(peer), Error]), - stop({shutdown, keepalive_error}, State#client_state{keepalive = undefined}) + ?WSLOG(warning, "Keepalive error - ~p", [Error], Req), + shutdown(keepalive_error, State) end; -handle_info({'EXIT', WsPid, Reason}, State = #client_state{ws_pid = WsPid, - proto_state = ProtoState}) -> - ClientId = emqttd_protocol:clientid(ProtoState), - lager:warning("Websocket client ~s exit: reason=~p", [ClientId, Reason]), - stop({shutdown, websocket_closed}, State); +handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{ws_pid = WsPid}) -> + stop(Reason, State); -handle_info(Info, State = #client_state{request = Req}) -> - lager:error("Client(WebSocket) ~s: Unexpected Info - ~p", [Req:get(peer), Info]), +handle_info(Info, State = #wsclient_state{request = Req}) -> + ?WSLOG(error, "Unexpected Info: ~p", [Info], Req), noreply(State). -terminate(Reason, #client_state{proto_state = ProtoState, keepalive = KeepAlive}) -> - lager:info("WebSocket client terminated: ~p", [Reason]), +terminate(Reason, #wsclient_state{proto_state = ProtoState, keepalive = KeepAlive}) -> emqttd_keepalive:cancel(KeepAlive), case Reason of {shutdown, Error} -> @@ -232,12 +246,19 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions %%%============================================================================= +with_proto_state(Fun, State = #wsclient_state{proto_state = ProtoState}) -> + {ok, ProtoState1} = Fun(ProtoState), + noreply(State#wsclient_state{proto_state = ProtoState1}). + +with_session(Fun, State = #wsclient_state{proto_state = ProtoState}) -> + Fun(emqttd_protocol:session(ProtoState)), noreply(State). + noreply(State) -> {noreply, State, hibernate}. +shutdown(Reason, State) -> + stop({shutdown, Reason}, State). + stop(Reason, State ) -> {stop, Reason, State}. -with_session(Fun, State = #client_state{proto_state = ProtoState}) -> - Fun(emqttd_protocol:session(ProtoState)), noreply(State). - From e538d91efcd109ac832386a5e783f703ae08f849 Mon Sep 17 00:00:00 2001 From: Feng Date: Sun, 1 Nov 2015 12:47:24 +0800 Subject: [PATCH 25/37] esockd3 --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 4b39a2f2b..62cdedb62 100644 --- a/rebar.config +++ b/rebar.config @@ -29,8 +29,8 @@ {deps, [ {gproc, ".*", {git, "git://github.com/uwiger/gproc.git", {branch, "master"}}}, {lager, ".*", {git, "git://github.com/basho/lager.git", {branch, "master"}}}, - {esockd, "3.*", {git, "git://github.com/emqtt/esockd.git", {branch, "3.0"}}}, - {mochiweb, ".*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "4.0"}}} + {esockd, ".*", {git, "git://github.com/emqtt/esockd.git", {branch, "esockd3"}}}, + {mochiweb, ".*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "mochiweb4"}}} ]}. {recursive_cmds, [ct, eunit, clean]}. From c7f0e336743b67a582776357528c8fd53e317b99 Mon Sep 17 00:00:00 2001 From: Feng Date: Sun, 1 Nov 2015 16:03:32 +0800 Subject: [PATCH 26/37] rate limit --- src/emqttd.erl | 11 +---------- src/emqttd_client.erl | 12 ++++++------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/emqttd.erl b/src/emqttd.erl index 09661d6c8..eeb5bab4d 100644 --- a/src/emqttd.erl +++ b/src/emqttd.erl @@ -91,8 +91,7 @@ open_listener({https, Port, Options}) -> mochiweb:start_http(Port, Options, MFArgs). open_listener(Protocol, Port, Options) -> - Rl = rate_limiter(emqttd_opts:g(rate_limit, Options)), - MFArgs = {emqttd_client, start_link, [[{rate_limiter, Rl} | env(mqtt)]]}, + MFArgs = {emqttd_client, start_link, [env(mqtt)]}, esockd:open(Protocol, Port, merge_sockopts(Options) , MFArgs). merge_sockopts(Options) -> @@ -100,14 +99,6 @@ merge_sockopts(Options) -> proplists:get_value(sockopts, Options, [])), emqttd_opts:merge(Options, [{sockopts, SockOpts}]). -%% TODO: will refactor in 0.14.0 release. -rate_limiter(undefined) -> - undefined; -rate_limiter(Config) -> - Bps = fun(S) -> list_to_integer(string:strip(S)) * 1024 end, - [Burst, Rate] = [Bps(S) || S <- string:tokens(Config, ",")], - esockd_rate_limiter:new(Burst, Rate). - %%------------------------------------------------------------------------------ %% @doc Close Listeners %% @end diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index f471d311b..23c53d44b 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -281,14 +281,14 @@ received(Bytes, State = #client_state{parser_fun = ParserFun, rate_limit(_Size, State = #client_state{rate_limit = undefined}) -> run_socket(State); -rate_limit(Size, State = #client_state{rate_limit = Limiter}) -> - case esockd_ratelimit:check(Limiter, Size) of - {0, Limiter1} -> - run_socket(State#client_state{conn_state = running, rate_limit = Limiter1}); - {Pause, Limiter1} -> +rate_limit(Size, State = #client_state{rate_limit = Rl}) -> + case Rl:check(Size) of + {0, Rl1} -> + run_socket(State#client_state{conn_state = running, rate_limit = Rl1}); + {Pause, Rl1} -> ?LOG(error, "Rate limiter pause for ~p", [Size, Pause], State), erlang:send_after(Pause, self(), activate_sock), - State#client_state{conn_state = blocked, rate_limit = Limiter1} + State#client_state{conn_state = blocked, rate_limit = Rl1} end. run_socket(State = #client_state{conn_state = blocked}) -> From 4b26291cb9aa8489404f767490018f6d9b7656f8 Mon Sep 17 00:00:00 2001 From: Feng Date: Sun, 1 Nov 2015 20:44:21 +0800 Subject: [PATCH 27/37] client(State) --- src/emqttd_protocol.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index 954332afe..e9395ed3f 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -182,7 +182,7 @@ process(Packet = ?CONNECT_PACKET(Var), State0) -> send(?CONNACK_PACKET(ReturnCode1), State3); process(Packet = ?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload), State) -> - case check_acl(publish, Topic, State) of + case check_acl(publish, Topic, client(State)) of allow -> publish(Packet, State); deny -> @@ -210,7 +210,8 @@ process(?SUBSCRIBE_PACKET(PacketId, []), State) -> send(?SUBACK_PACKET(PacketId, []), State); process(?SUBSCRIBE_PACKET(PacketId, TopicTable), State = #proto_state{session = Session}) -> - AllowDenies = [check_acl(subscribe, Topic, State) || {Topic, _Qos} <- TopicTable], + Client = client(State), + AllowDenies = [check_acl(subscribe, Topic, Client) || {Topic, _Qos} <- TopicTable], case lists:member(deny, AllowDenies) of true -> ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicTable], State), @@ -391,16 +392,16 @@ validate_qos(_) -> false. %% PUBLISH ACL is cached in process dictionary. -check_acl(publish, Topic, State) -> +check_acl(publish, Topic, Client) -> case get({acl, publish, Topic}) of undefined -> - AllowDeny = emqttd_access_control:check_acl(client(State), publish, Topic), + AllowDeny = emqttd_access_control:check_acl(Client, publish, Topic), put({acl, publish, Topic}, AllowDeny), AllowDeny; AllowDeny -> AllowDeny end; -check_acl(subscribe, Topic, State) -> - emqttd_access_control:check_acl(client(State), subscribe, Topic). +check_acl(subscribe, Topic, Client) -> + emqttd_access_control:check_acl(Client, subscribe, Topic). From 83d4801fe8f2694b4a04d6d9fff75f80feb37ed2 Mon Sep 17 00:00:00 2001 From: Feng Date: Sun, 1 Nov 2015 20:45:13 +0800 Subject: [PATCH 28/37] disc_copies --- src/emqttd_auth_username.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_auth_username.erl b/src/emqttd_auth_username.erl index 392d32961..855a3d6e6 100644 --- a/src/emqttd_auth_username.erl +++ b/src/emqttd_auth_username.erl @@ -105,7 +105,7 @@ init(Opts) -> mnesia:create_table(?AUTH_USERNAME_TAB, [ {disc_copies, [node()]}, {attributes, record_info(fields, ?AUTH_USERNAME_TAB)}]), - mnesia:add_table_copy(?AUTH_USERNAME_TAB, node(), ram_copies), + mnesia:add_table_copy(?AUTH_USERNAME_TAB, node(), disc_copies), emqttd_ctl:register_cmd(users, {?MODULE, cli}, []), {ok, Opts}. From 3c69d1a5c1641e5d16ed0f63b0a8c1a0df8956ba Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 2 Nov 2015 15:00:43 +0800 Subject: [PATCH 29/37] fix log --- src/emqttd_cm.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_cm.erl b/src/emqttd_cm.erl index 9f5462b57..1347e1815 100644 --- a/src/emqttd_cm.erl +++ b/src/emqttd_cm.erl @@ -126,7 +126,7 @@ handle_cast({unregister, ClientId, Pid}, State) -> [_] -> ignore; [] -> - ?LOG(error, "Cannot find registered: ~p", [Pid], State) + lager:warning("CM(~s): Cannot find registered pid ~p", [ClientId, Pid]) end, {noreply, setstats(State)}; From a50df24aabaa2d86aa4853cddfaf0500f6594873 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 2 Nov 2015 15:32:06 +0800 Subject: [PATCH 30/37] 100 --- rel/files/emqttd.config.production | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/files/emqttd.config.production b/rel/files/emqttd.config.production index a657d7bf3..d96cb1373 100644 --- a/rel/files/emqttd.config.production +++ b/rel/files/emqttd.config.production @@ -246,7 +246,7 @@ {long_gc, false}, %% Long Schedule(ms) - {long_schedule, 50}, + {long_schedule, 100}, %% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. %% 8 * 1024 * 1024 From cff4ca029690e5cf283782b0e61ca827c8cddce1 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 2 Nov 2015 15:51:24 +0800 Subject: [PATCH 31/37] vsn --- src/emqttd_cli.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqttd_cli.erl b/src/emqttd_cli.erl index fd70ce4f6..59441d516 100644 --- a/src/emqttd_cli.erl +++ b/src/emqttd_cli.erl @@ -72,8 +72,8 @@ status([]) -> case lists:keysearch(emqttd, 1, application:which_applications()) of false -> ?PRINT_MSG("emqttd is not running~n"); - {value,_Version} -> - ?PRINT_MSG("emqttd is running~n") + {value, {emqttd, _Desc, Vsn}} -> + ?PRINT("emqttd ~s is running~n", [Vsn]) end; status(_) -> ?PRINT_CMD("status", "query broker status"). From ecc0f052ef9abee8fa8db10b1f302976aef0f1ac Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 2 Nov 2015 15:51:34 +0800 Subject: [PATCH 32/37] reply --- src/emqttd_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 23c53d44b..3ad5a3728 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -124,7 +124,7 @@ handle_call(info, _From, State = #client_state{connection = Connection, ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS), ProtoInfo = emqttd_protocol:info(ProtoState), {ok, SockStats} = Connection:getstat(?SOCK_STATS), - {noreply, lists:append([ClientInfo, [{proto_info, ProtoInfo}, + {reply, lists:append([ClientInfo, [{proto_info, ProtoInfo}, {sock_stats, SockStats}]]), State}; handle_call(kick, _From, State) -> From ea564ec8ef9b3276355c77f814195cd663d34715 Mon Sep 17 00:00:00 2001 From: Feng Date: Mon, 2 Nov 2015 20:34:23 +0800 Subject: [PATCH 33/37] mod --- src/emqttd_app.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqttd_app.erl b/src/emqttd_app.erl index 53adfa582..4ee9cabd1 100644 --- a/src/emqttd_app.erl +++ b/src/emqttd_app.erl @@ -82,7 +82,7 @@ start_servers(Sup) -> {"emqttd session supervisor", {supervisor, emqttd_session_sup}}, {"emqttd broker", emqttd_broker}, {"emqttd alarm", emqttd_alarm}, - {"emqttd mode supervisor", emqttd_mod_sup}, + {"emqttd mod supervisor", emqttd_mod_sup}, {"emqttd bridge supervisor", {supervisor, emqttd_bridge_sup}}, {"emqttd access control", emqttd_access_control}, {"emqttd system monitor", emqttd_sysmon, emqttd:env(sysmon)}], From 9bad2bf65b604c9e48009396addc2973de7c1619 Mon Sep 17 00:00:00 2001 From: Feng Date: Tue, 3 Nov 2015 12:52:27 +0800 Subject: [PATCH 34/37] support Seq --- src/emqttd_access_control.erl | 36 ++++++++++++++++------------ test/emqttd_access_control_tests.erl | 21 +++++++++------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/emqttd_access_control.erl b/src/emqttd_access_control.erl index 320b6f75f..7d15d5904 100644 --- a/src/emqttd_access_control.erl +++ b/src/emqttd_access_control.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_access_control). -author("Feng Lee "). @@ -36,14 +35,13 @@ -define(SERVER, ?MODULE). %% API Function Exports --export([start_link/0, - start_link/1, +-export([start_link/0, start_link/1, auth/2, % authentication check_acl/3, % acl check reload_acl/0, % reload acl - register_mod/3, - unregister_mod/2, lookup_mods/1, + register_mod/3, register_mod/4, + unregister_mod/2, stop/0]). %% gen_server callbacks @@ -77,7 +75,7 @@ auth(Client, Password) when is_record(Client, mqtt_client) -> auth(Client, Password, lookup_mods(auth)). auth(_Client, _Password, []) -> {error, "No auth module to check!"}; -auth(Client, Password, [{Mod, State} | Mods]) -> +auth(Client, Password, [{Mod, State, _Seq} | Mods]) -> case Mod:check(Client, Password, State) of ok -> ok; {error, Reason} -> {error, Reason}; @@ -100,7 +98,7 @@ check_acl(Client, PubSub, Topic) when ?IS_PUBSUB(PubSub) -> check_acl(#mqtt_client{client_id = ClientId}, PubSub, Topic, []) -> lager:error("ACL: nomatch when ~s ~s ~s", [ClientId, PubSub, Topic]), allow; -check_acl(Client, PubSub, Topic, [{M, State}|AclMods]) -> +check_acl(Client, PubSub, Topic, [{M, State, _Seq}|AclMods]) -> case M:check_acl({Client, PubSub, Topic}, State) of allow -> allow; deny -> deny; @@ -113,7 +111,7 @@ check_acl(Client, PubSub, Topic, [{M, State}|AclMods]) -> %%------------------------------------------------------------------------------ -spec reload_acl() -> list() | {error, any()}. reload_acl() -> - [M:reload_acl(State) || {M, State} <- lookup_mods(acl)]. + [M:reload_acl(State) || {M, State, _Seq} <- lookup_mods(acl)]. %%------------------------------------------------------------------------------ %% @doc Register authentication or ACL module @@ -121,7 +119,11 @@ reload_acl() -> %%------------------------------------------------------------------------------ -spec register_mod(Type :: auth | acl, Mod :: atom(), Opts :: list()) -> ok | {error, any()}. register_mod(Type, Mod, Opts) when Type =:= auth; Type =:= acl-> - gen_server:call(?SERVER, {register_mod, Type, Mod, Opts}). + register_mod(Type, Mod, Opts, 0). + +-spec register_mod(auth | acl, atom(), list(), pos_integer()) -> ok | {error, any()}. +register_mod(Type, Mod, Opts, Seq) when Type =:= auth; Type =:= acl-> + gen_server:call(?SERVER, {register_mod, Type, Mod, Opts, Seq}). %%------------------------------------------------------------------------------ %% @doc Unregister authentication or ACL module @@ -172,22 +174,26 @@ init_mods(acl, AclMods) -> init_mod(Fun, Name, Opts) -> Module = Fun(Name), {ok, State} = Module:init(Opts), - {Module, State}. + {Module, State, 0}. -handle_call({register_mod, Type, Mod, Opts}, _From, State) -> +handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) -> Mods = lookup_mods(Type), Reply = case lists:keyfind(Mod, 1, Mods) of - false -> + false -> case catch Mod:init(Opts) of - {ok, ModState} -> - ets:insert(?ACCESS_CONTROL_TAB, {tab_key(Type), [{Mod, ModState}|Mods]}), + {ok, ModState} -> + NewMods = + lists:sort(fun({_, _, Seq1}, {_, _, Seq2}) -> + Seq1 >= Seq2 + end, [{Mod, ModState, Seq} | Mods]), + ets:insert(?ACCESS_CONTROL_TAB, {tab_key(Type), NewMods}), ok; {'EXIT', Error} -> lager:error("Access Control: register ~s error - ~p", [Mod, Error]), {error, Error} end; - _ -> + _ -> {error, existed} end, {reply, Reply, State}; diff --git a/test/emqttd_access_control_tests.erl b/test/emqttd_access_control_tests.erl index 7db0490ef..5da45f4a8 100644 --- a/test/emqttd_access_control_tests.erl +++ b/test/emqttd_access_control_tests.erl @@ -42,30 +42,33 @@ register_mod_test() -> with_acl( fun() -> emqttd_access_control:register_mod(acl, emqttd_acl_test_mod, []), - ?assertMatch([{emqttd_acl_test_mod, _}, {emqttd_acl_internal, _}], + ?assertMatch([{emqttd_acl_test_mod, _, 0}, {emqttd_acl_internal, _, 0}], emqttd_access_control:lookup_mods(acl)), emqttd_access_control:register_mod(auth, emqttd_auth_anonymous_test_mod,[]), - ?assertMatch([{emqttd_auth_anonymous_test_mod, _}, {emqttd_auth_anonymous, _}], - emqttd_access_control:lookup_mods(auth)) + emqttd_access_control:register_mod(auth, emqttd_auth_dashboard, [], 99), + ?assertMatch([{emqttd_auth_dashboard, _, 99}, + {emqttd_auth_anonymous_test_mod, _, 0}, + {emqttd_auth_anonymous, _, 0}], + emqttd_access_control:lookup_mods(auth)) end). unregister_mod_test() -> with_acl( fun() -> - emqttd_access_control:register_mod(acl,emqttd_acl_test_mod, []), - ?assertMatch([{emqttd_acl_test_mod, _}, {emqttd_acl_internal, _}], + emqttd_access_control:register_mod(acl, emqttd_acl_test_mod, []), + ?assertMatch([{emqttd_acl_test_mod, _, 0}, {emqttd_acl_internal, _, 0}], emqttd_access_control:lookup_mods(acl)), emqttd_access_control:unregister_mod(acl, emqttd_acl_test_mod), timer:sleep(5), - ?assertMatch([{emqttd_acl_internal, _}], emqttd_access_control:lookup_mods(acl)), + ?assertMatch([{emqttd_acl_internal, _, 0}], emqttd_access_control:lookup_mods(acl)), emqttd_access_control:register_mod(auth, emqttd_auth_anonymous_test_mod,[]), - ?assertMatch([{emqttd_auth_anonymous_test_mod, _}, {emqttd_auth_anonymous, _}], + ?assertMatch([{emqttd_auth_anonymous_test_mod, _, 0}, {emqttd_auth_anonymous, _, 0}], emqttd_access_control:lookup_mods(auth)), emqttd_access_control:unregister_mod(auth, emqttd_auth_anonymous_test_mod), timer:sleep(5), - ?assertMatch([{emqttd_auth_anonymous, _}], emqttd_access_control:lookup_mods(auth)) + ?assertMatch([{emqttd_auth_anonymous, _, 0}], emqttd_access_control:lookup_mods(auth)) end). check_acl_test() -> @@ -83,7 +86,7 @@ check_acl_test() -> with_acl(Fun) -> process_flag(trap_exit, true), - AclOpts = [ + AclOpts = [ {auth, [ %% Authentication with username, password %{username, []}, From d09721301e18ccdd03a6718d9aeeddc94d11de1c Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 4 Nov 2015 10:22:49 +0800 Subject: [PATCH 35/37] conflict --- src/emqttd_client.erl | 2 +- src/emqttd_protocol.erl | 2 +- src/emqttd_ws_client.erl | 2 +- test/emqttd_auth_dashboard.erl | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 test/emqttd_auth_dashboard.erl diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 3ad5a3728..a684737a5 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -170,7 +170,7 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State) -> handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State), - shutdown(confict, State); + shutdown(conflict, State); handle_info(activate_sock, State) -> noreply(run_socket(State#client_state{conn_state = running})); diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index e9395ed3f..33571e213 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -282,7 +282,7 @@ redeliver({?PUBREL, PacketId}, State) -> shutdown(_Error, #proto_state{client_id = undefined}) -> ignore; -shutdown(confict, #proto_state{client_id = ClientId}) -> +shutdown(conflict, #proto_state{client_id = ClientId}) -> emqttd_cm:unregister(ClientId); shutdown(Error, State = #proto_state{client_id = ClientId, will_msg = WillMsg}) -> diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 1d02ab46c..cb9a49e74 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -196,7 +196,7 @@ handle_info({redeliver, {?PUBREL, PacketId}}, State) -> handle_info({shutdown, conflict, {ClientId, NewPid}}, State = #wsclient_state{request = Req}) -> ?WSLOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], Req), - shutdown(confict, State); + shutdown(conflict, State); handle_info({keepalive, start, Interval}, State = #wsclient_state{request = Req}) -> ?WSLOG(debug, "Keepalive at the interval of ~p", [Interval], Req), diff --git a/test/emqttd_auth_dashboard.erl b/test/emqttd_auth_dashboard.erl new file mode 100644 index 000000000..ea9aca7e1 --- /dev/null +++ b/test/emqttd_auth_dashboard.erl @@ -0,0 +1,14 @@ + +-module(emqttd_auth_dashboard). + +%% Auth callbacks +-export([init/1, check/3, description/0]). + +init(Opts) -> + {ok, Opts}. + +check(_Client, _Password, _Opts) -> + allow. + +description() -> + "Test emqttd_auth_dashboard Mod". From 8491467bbbfb39e7de5d846e96341c405e63552c Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 4 Nov 2015 21:42:17 +0800 Subject: [PATCH 36/37] support 'and', 'or' --- rel/files/acl.config | 34 +++++++++++++++---------------- src/emqttd_access_rule.erl | 24 ++++++++++++++++++---- test/emqttd_access_rule_tests.erl | 21 +++++++++++++++---- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/rel/files/acl.config b/rel/files/acl.config index 3359b2b04..9b1d512a6 100644 --- a/rel/files/acl.config +++ b/rel/files/acl.config @@ -1,21 +1,21 @@ %%%----------------------------------------------------------------------------- -%% -%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) -%% -%% -type who() :: all | binary() | -%% {ipaddr, esockd_access:cidr()} | -%% {client, binary()} | -%% {user, binary()}. -%% -%% -type access() :: subscribe | publish | pubsub. -%% -%% -type topic() :: binary(). -%% -%% -type rule() :: {allow, all} | -%% {allow, who(), access(), list(topic())} | -%% {deny, all} | -%% {deny, who(), access(), list(topic())}. -%% +%%% +%%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) +%%% +%%% -type who() :: all | binary() | +%%% {ipaddr, esockd_access:cidr()} | +%%% {client, binary()} | +%%% {user, binary()}. +%%% +%%% -type access() :: subscribe | publish | pubsub. +%%% +%%% -type topic() :: binary(). +%%% +%%% -type rule() :: {allow, all} | +%%% {allow, who(), access(), list(topic())} | +%%% {deny, all} | +%%% {deny, who(), access(), list(topic())}. +%%% %%%----------------------------------------------------------------------------- {allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. diff --git a/src/emqttd_access_rule.erl b/src/emqttd_access_rule.erl index fab9461b9..30ed8f87b 100644 --- a/src/emqttd_access_rule.erl +++ b/src/emqttd_access_rule.erl @@ -24,7 +24,6 @@ %%% %%% @end %%%----------------------------------------------------------------------------- - -module(emqttd_access_rule). -author("Feng Lee "). @@ -49,17 +48,22 @@ -export([compile/1, match/3]). +-define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= deny))). + %%------------------------------------------------------------------------------ %% @doc Compile access rule %% @end %%------------------------------------------------------------------------------ -compile({A, all}) when (A =:= allow) orelse (A =:= deny) -> +compile({A, all}) when ?ALLOW_DENY(A) -> {A, all}; -compile({A, Who, Access, TopicFilters}) when (A =:= allow) orelse (A =:= deny) -> +compile({A, Who, Access, Topic}) when ?ALLOW_DENY(A) andalso is_binary(Topic) -> + {A, compile(who, Who), Access, [compile(topic, Topic)]}; + +compile({A, Who, Access, TopicFilters}) when ?ALLOW_DENY(A) -> {A, compile(who, Who), Access, [compile(topic, Topic) || Topic <- TopicFilters]}. -compile(who, all) -> +compile(who, all) -> all; compile(who, {ipaddr, CIDR}) -> {Start, End} = esockd_access:range(CIDR), @@ -72,6 +76,10 @@ compile(who, {user, all}) -> {user, all}; compile(who, {user, Username}) -> {user, bin(Username)}; +compile(who, {'and', Conds}) when is_list(Conds) -> + {'and', [compile(who, Cond) || Cond <- Conds]}; +compile(who, {'or', Conds}) when is_list(Conds) -> + {'or', [compile(who, Cond) || Cond <- Conds]}; compile(topic, {eq, Topic}) -> {eq, emqttd_topic:words(bin(Topic))}; @@ -120,6 +128,14 @@ match_who(#mqtt_client{peername = undefined}, {ipaddr, _Tup}) -> match_who(#mqtt_client{peername = {IP, _}}, {ipaddr, {_CDIR, Start, End}}) -> I = esockd_access:atoi(IP), I >= Start andalso I =< End; +match_who(Client, {'and', Conds}) when is_list(Conds) -> + lists:foldl(fun(Who, Allow) -> + match_who(Client, Who) andalso Allow + end, true, Conds); +match_who(Client, {'or', Conds}) when is_list(Conds) -> + lists:foldl(fun(Who, Allow) -> + match_who(Client, Who) orelse Allow + end, false, Conds); match_who(_Client, _Who) -> false. diff --git a/test/emqttd_access_rule_tests.erl b/test/emqttd_access_rule_tests.erl index 142beeaeb..f46f23ce4 100644 --- a/test/emqttd_access_rule_tests.erl +++ b/test/emqttd_access_rule_tests.erl @@ -35,6 +35,14 @@ -include_lib("eunit/include/eunit.hrl"). compile_test() -> + + ?assertMatch({allow, {'and', [{ipaddr, {"127.0.0.1", _I, _I}}, + {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]}, + compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]})), + ?assertMatch({allow, {'or', [{ipaddr, {"127.0.0.1", _I, _I}}, + {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]}, + compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]})), + ?assertMatch({allow, {ipaddr, {"127.0.0.1", _I, _I}}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]}, compile({allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]})), ?assertMatch({allow, {user, <<"testuser">>}, subscribe, [ [<<"a">>, <<"b">>, <<"c">>], [<<"d">>, <<"e">>, <<"f">>, '#'] ]}, @@ -69,10 +77,15 @@ match_test() -> ?assertMatch({matched, allow}, match(User, <<"clients/testClient">>, compile({allow, all, pubsub, ["clients/$c"]}))), ?assertMatch({matched, allow}, match(#mqtt_client{username = <<"user2">>}, <<"users/user2/abc/def">>, - compile({allow, all, subscribe, ["users/$u/#"]}))), - ?assertMatch({matched, deny}, - match(User, <<"d/e/f">>, - compile({deny, all, subscribe, ["$SYS/#", "#"]}))). + compile({allow, all, subscribe, ["users/$u/#"]}))), + ?assertMatch({matched, deny}, match(User, <<"d/e/f">>, + compile({deny, all, subscribe, ["$SYS/#", "#"]}))), + Rule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, <<"Topic">>}), + ?assertMatch(nomatch, match(User, <<"Topic">>, Rule)), + AndRule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"TestUser">>}]}, publish, <<"Topic">>}), + ?assertMatch({matched, allow}, match(User, <<"Topic">>, AndRule)), + OrRule = compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, ["Topic"]}), + ?assertMatch({matched, allow}, match(User, <<"Topic">>, OrRule)). -endif. From 5aa30f0b1cffedf336b3cee2ae2d3efb714aeb9b Mon Sep 17 00:00:00 2001 From: Feng Date: Wed, 4 Nov 2015 22:23:20 +0800 Subject: [PATCH 37/37] master --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 62cdedb62..84968128f 100644 --- a/rebar.config +++ b/rebar.config @@ -29,8 +29,8 @@ {deps, [ {gproc, ".*", {git, "git://github.com/uwiger/gproc.git", {branch, "master"}}}, {lager, ".*", {git, "git://github.com/basho/lager.git", {branch, "master"}}}, - {esockd, ".*", {git, "git://github.com/emqtt/esockd.git", {branch, "esockd3"}}}, - {mochiweb, ".*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "mochiweb4"}}} + {esockd, "3.*", {git, "git://github.com/emqtt/esockd.git", {branch, "master"}}}, + {mochiweb, "4.*", {git, "git://github.com/emqtt/mochiweb.git", {branch, "master"}}} ]}. {recursive_cmds, [ct, eunit, clean]}.