From b4d981daf2a549554e8e76f88b9b3c0676779932 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 4 Dec 2018 15:59:24 +0800 Subject: [PATCH 01/54] Add a sequence module to generate index for subscription sharding --- Makefile | 2 +- src/emqx_sequence.erl | 58 ++++++++++++++++++++++++++++++++++++ test/emqx_sequence_SUITE.erl | 37 +++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/emqx_sequence.erl create mode 100644 test/emqx_sequence_SUITE.erl diff --git a/Makefile b/Makefile index 26bcf22ce..2c1693813 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ - emqx_hooks emqx_batch + emqx_hooks emqx_batch emqx_sequence CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl new file mode 100644 index 000000000..62a882294 --- /dev/null +++ b/src/emqx_sequence.erl @@ -0,0 +1,58 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence). + +-export([create/0, create/1]). +-export([generate/1, generate/2]). +-export([reclaim/1, reclaim/2]). + +-type(key() :: term()). +-type(seqid() :: non_neg_integer()). + +-define(DEFAULT_TAB, ?MODULE). + +%% @doc Create a sequence. +-spec(create() -> ok). +create() -> + create(?DEFAULT_TAB). + +-spec(create(atom()) -> ok). +create(Tab) -> + _ = ets:new(Tab, [set, public, named_table, {write_concurrency, true}]), + ok. + +%% @doc Generate a sequence id. +-spec(generate(key()) -> seqid()). +generate(Key) -> + generate(?DEFAULT_TAB, Key). + +-spec(generate(atom(), key()) -> seqid()). +generate(Tab, Key) -> + ets:update_counter(Tab, Key, {2, 1}, {Key, 0}). + +%% @doc Reclaim a sequence id. +-spec(reclaim(key()) -> seqid()). +reclaim(Key) -> + reclaim(?DEFAULT_TAB, Key). + +-spec(reclaim(atom(), key()) -> seqid()). +reclaim(Tab, Key) -> + try ets:update_counter(Tab, Key, {2, -1, 0, 0}) of + 0 -> ets:delete_object(Tab, {Key, 0}), 0; + I -> I + catch + error:badarg -> 0 + end. + diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl new file mode 100644 index 000000000..999a95723 --- /dev/null +++ b/test/emqx_sequence_SUITE.erl @@ -0,0 +1,37 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-import(emqx_sequence, [generate/1, reclaim/1]). + +all() -> + [sequence_generate]. + +sequence_generate(_) -> + ok = emqx_sequence:create(), + ?assertEqual(1, generate(key)), + ?assertEqual(2, generate(key)), + ?assertEqual(3, generate(key)), + ?assertEqual(2, reclaim(key)), + ?assertEqual(1, reclaim(key)), + ?assertEqual(0, reclaim(key)), + ?assertEqual(false, ets:member(emqx_sequence, key)), + ?assertEqual(1, generate(key)). + From bce1ddc5c4d226e7f0e1ae6648ab5d1da54c12ff Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 6 Dec 2018 18:45:07 +0800 Subject: [PATCH 02/54] Implement a hash-based subscription sharding --- src/emqx.erl | 53 ++--- src/emqx_broker.erl | 444 ++++++++++++++--------------------- src/emqx_broker_helper.erl | 75 +++--- src/emqx_broker_sup.erl | 51 ++-- src/emqx_local_bridge.erl | 2 +- src/emqx_sequence.erl | 54 +++-- src/emqx_session.erl | 2 +- test/emqx_sequence_SUITE.erl | 20 +- 8 files changed, 299 insertions(+), 402 deletions(-) diff --git a/src/emqx.erl b/src/emqx.erl index 72f1d6f81..3792cc4f8 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -26,7 +26,6 @@ %% PubSub management API -export([topics/0, subscriptions/1, subscribers/1, subscribed/2]). --export([get_subopts/2, set_subopts/3]). %% Hooks API -export([hook/2, hook/3, hook/4, unhook/2, run_hooks/2, run_hooks/3]). @@ -70,20 +69,18 @@ is_running(Node) -> subscribe(Topic) -> emqx_broker:subscribe(iolist_to_binary(Topic)). --spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok). +-spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId)-> emqx_broker:subscribe(iolist_to_binary(Topic), SubId); -subscribe(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:subscribe(iolist_to_binary(Topic), SubPid). +subscribe(Topic, SubOpts) when is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubOpts). --spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid(), - emqx_types:subopts()) -> ok). -subscribe(Topic, SubId, Options) when is_atom(SubId); is_binary(SubId)-> - emqx_broker:subscribe(iolist_to_binary(Topic), SubId, Options); -subscribe(Topic, SubPid, Options) when is_pid(SubPid)-> - emqx_broker:subscribe(iolist_to_binary(Topic), SubPid, Options). +-spec(subscribe(emqx_topic:topic() | string(), + emqx_types:subid() | pid(), emqx_types:subopts()) -> ok). +subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubId, SubOpts). --spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}). +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). publish(Msg) -> emqx_broker:publish(Msg). @@ -91,26 +88,14 @@ publish(Msg) -> unsubscribe(Topic) -> emqx_broker:unsubscribe(iolist_to_binary(Topic)). --spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok). -unsubscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId); -unsubscribe(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubPid). +-spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid()) -> ok). +unsubscribe(Topic, SubId) -> + emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId). %%------------------------------------------------------------------------------ %% PubSub management API %%------------------------------------------------------------------------------ --spec(get_subopts(emqx_topic:topic() | string(), emqx_types:subscriber()) - -> emqx_types:subopts()). -get_subopts(Topic, Subscriber) -> - emqx_broker:get_subopts(iolist_to_binary(Topic), Subscriber). - --spec(set_subopts(emqx_topic:topic() | string(), emqx_types:subscriber(), - emqx_types:subopts()) -> boolean()). -set_subopts(Topic, Subscriber, Options) when is_map(Options) -> - emqx_broker:set_subopts(iolist_to_binary(Topic), Subscriber, Options). - -spec(topics() -> list(emqx_topic:topic())). topics() -> emqx_router:topics(). @@ -118,15 +103,15 @@ topics() -> emqx_router:topics(). subscribers(Topic) -> emqx_broker:subscribers(iolist_to_binary(Topic)). --spec(subscriptions(emqx_types:subscriber()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(Subscriber) -> - emqx_broker:subscriptions(Subscriber). +-spec(subscriptions(pid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). +subscriptions(SubPid) when is_pid(SubPid) -> + emqx_broker:subscriptions(SubPid). --spec(subscribed(emqx_topic:topic() | string(), pid() | emqx_types:subid()) -> boolean()). -subscribed(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:subscribed(iolist_to_binary(Topic), SubPid); -subscribed(Topic, SubId) when is_atom(SubId); is_binary(SubId) -> - emqx_broker:subscribed(iolist_to_binary(Topic), SubId). +-spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic() | string()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + emqx_broker:subscribed(SubPid, iolist_to_binary(Topic)); +subscribed(SubId, Topic) when is_atom(SubId); is_binary(SubId) -> + emqx_broker:subscribed(SubId, iolist_to_binary(Topic)). %%------------------------------------------------------------------------------ %% Hooks API diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index ca4a86c87..3a3cde1fa 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -19,150 +19,130 @@ -include("emqx.hrl"). -export([start_link/2]). --export([subscribe/1, subscribe/2, subscribe/3, subscribe/4]). --export([multi_subscribe/1, multi_subscribe/2, multi_subscribe/3]). +-export([subscribe/1, subscribe/2, subscribe/3]). +-export([unsubscribe/1, unsubscribe/2]). +-export([subscriber_down/1]). -export([publish/1, safe_publish/1]). --export([unsubscribe/1, unsubscribe/2, unsubscribe/3]). --export([multi_unsubscribe/1, multi_unsubscribe/2, multi_unsubscribe/3]). -export([dispatch/2, dispatch/3]). -export([subscriptions/1, subscribers/1, subscribed/2]). --export([get_subopts/2, set_subopts/3]). +-export([get_subopts/2, set_subopts/2]). -export([topics/0]). +%% Stats fun +-export([stats_fun/0]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - --record(state, {pool, id, submap, submon}). --record(subscribe, {topic, subpid, subid, subopts = #{}}). --record(unsubscribe, {topic, subpid, subid}). - -%% The default request timeout +-define(SHARD, 1024). -define(TIMEOUT, 60000). -define(BROKER, ?MODULE). %% ETS tables --define(SUBOPTION, emqx_suboption). --define(SUBSCRIBER, emqx_subscriber). +-define(SUBID, emqx_subid). +-define(SUBOPTION, emqx_suboption). +-define(SUBSCRIBER, emqx_subscriber). -define(SUBSCRIPTION, emqx_subscription). +%% Gards -define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, - [Pool, Id], [{hibernate_after, 2000}]). + _ = create_tabs(), + gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], []). %%------------------------------------------------------------------------------ -%% Subscribe +%% Create tabs +%%------------------------------------------------------------------------------ + +create_tabs() -> + TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], + + %% SubId: SubId -> SubPid1, SubPid2,... + _ = emqx_tables:new(?SUBID, [bag | TabOpts]), + %% SubOption: {SubPid, Topic} -> SubOption + _ = emqx_tables:new(?SUBOPTION, [set | TabOpts]), + + %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... + %% duplicate_bag: o(1) insert + _ = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + + %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... + %% duplicate_bag: o(1) insert + emqx_tables:new(?SUBSCRIBER, [duplicate_bag | TabOpts]). + +%%------------------------------------------------------------------------------ +%% Subscribe API %%------------------------------------------------------------------------------ -spec(subscribe(emqx_topic:topic()) -> ok). subscribe(Topic) when is_binary(Topic) -> - subscribe(Topic, self()). + subscribe(Topic, undefined). --spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok). -subscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - subscribe(Topic, SubPid, undefined); +-spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - subscribe(Topic, self(), SubId). + subscribe(Topic, SubId, #{}); +subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> + subscribe(Topic, undefined, SubOpts). --spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid(), - emqx_types:subid() | emqx_types:subopts()) -> ok). -subscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - subscribe(Topic, SubPid, SubId, #{qos => 0}); -subscribe(Topic, SubPid, SubOpts) when is_binary(Topic), is_pid(SubPid), is_map(SubOpts) -> - subscribe(Topic, SubPid, undefined, SubOpts); +-spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts) -> - subscribe(Topic, self(), SubId, SubOpts). - --spec(subscribe(emqx_topic:topic(), pid(), emqx_types:subid(), emqx_types:subopts()) -> ok). -subscribe(Topic, SubPid, SubId, SubOpts) when is_binary(Topic), is_pid(SubPid), - ?is_subid(SubId), is_map(SubOpts) -> - Broker = pick(SubPid), - SubReq = #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}, - wait_for_reply(async_call(Broker, SubReq), ?TIMEOUT). - --spec(multi_subscribe(emqx_types:topic_table()) -> ok). -multi_subscribe(TopicTable) when is_list(TopicTable) -> - multi_subscribe(TopicTable, self()). - --spec(multi_subscribe(emqx_types:topic_table(), pid() | emqx_types:subid()) -> ok). -multi_subscribe(TopicTable, SubPid) when is_pid(SubPid) -> - multi_subscribe(TopicTable, SubPid, undefined); -multi_subscribe(TopicTable, SubId) when ?is_subid(SubId) -> - multi_subscribe(TopicTable, self(), SubId). - --spec(multi_subscribe(emqx_types:topic_table(), pid(), emqx_types:subid()) -> ok). -multi_subscribe(TopicTable, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - SubReq = fun(Topic, SubOpts) -> - #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts} - end, - wait_for_replies([async_call(Broker, SubReq(Topic, SubOpts)) - || {Topic, SubOpts} <- TopicTable], ?TIMEOUT). + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of + false -> + ok = emqx_broker_helper:monitor(SubPid, SubId), + Group = maps:get(share, SubOpts, undefined), + %% true = ets:insert(?SUBID, {SubId, SubPid}), + true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), + %% SeqId = emqx_broker_helper:create_seq(Topic), + true = ets:insert(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + ok = emqx_shared_sub:subscribe(Group, Topic, SubPid), + call(pick(Topic), {subscribe, Group, Topic}); + true -> ok + end. %%------------------------------------------------------------------------------ -%% Unsubscribe +%% Unsubscribe API %%------------------------------------------------------------------------------ -spec(unsubscribe(emqx_topic:topic()) -> ok). unsubscribe(Topic) when is_binary(Topic) -> - unsubscribe(Topic, self()). + SubPid = self(), + case ets:lookup(?SUBOPTION, {SubPid, Topic}) of + [{_, SubOpts}] -> + Group = maps:get(share, SubOpts, undefined), + true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), + true = ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid), + call(pick(Topic), {unsubscribe, Group, Topic}); + [] -> ok + end. --spec(unsubscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok). -unsubscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - unsubscribe(Topic, SubPid, undefined); -unsubscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - unsubscribe(Topic, self(), SubId). - --spec(unsubscribe(emqx_topic:topic(), pid(), emqx_types:subid()) -> ok). -unsubscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - UnsubReq = #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}, - wait_for_reply(async_call(Broker, UnsubReq), ?TIMEOUT). - --spec(multi_unsubscribe([emqx_topic:topic()]) -> ok). -multi_unsubscribe(Topics) -> - multi_unsubscribe(Topics, self()). - --spec(multi_unsubscribe([emqx_topic:topic()], pid() | emqx_types:subid()) -> ok). -multi_unsubscribe(Topics, SubPid) when is_pid(SubPid) -> - multi_unsubscribe(Topics, SubPid, undefined); -multi_unsubscribe(Topics, SubId) when ?is_subid(SubId) -> - multi_unsubscribe(Topics, self(), SubId). - --spec(multi_unsubscribe([emqx_topic:topic()], pid(), emqx_types:subid()) -> ok). -multi_unsubscribe(Topics, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - UnsubReq = fun(Topic) -> - #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId} - end, - wait_for_replies([async_call(Broker, UnsubReq(Topic)) || Topic <- Topics], ?TIMEOUT). +-spec(unsubscribe(emqx_topic:topic(), emqx_types:subid()) -> ok). +unsubscribe(Topic, _SubId) when is_binary(Topic) -> + unsubscribe(Topic). %%------------------------------------------------------------------------------ %% Publish %%------------------------------------------------------------------------------ --spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}). +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). publish(Msg) when is_record(Msg, message) -> _ = emqx_tracer:trace(publish, Msg), - {ok, case emqx_hooks:run('message.publish', [], Msg) of - {ok, Msg1 = #message{topic = Topic}} -> - Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)), - Delivery#delivery.results; - {stop, _} -> - emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]), - [] - end}. + case emqx_hooks:run('message.publish', [], Msg) of + {ok, Msg1 = #message{topic = Topic}} -> + Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)), + Delivery#delivery.results; + {stop, _} -> + emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]), + [] + end. --spec(safe_publish(emqx_types:message()) -> ok). %% Called internally +-spec(safe_publish(emqx_types:message()) -> ok). safe_publish(Msg) when is_record(Msg, message) -> try publish(Msg) @@ -227,98 +207,113 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> emqx_hooks:run('message.dropped', [#{node => node()}, Msg]), inc_dropped_cnt(Topic), Delivery; - [Sub] -> %% optimize? - dispatch(Sub, Topic, Msg), + [SubPid] -> %% optimize? + dispatch(SubPid, Topic, Msg), Delivery#delivery{results = [{dispatch, Topic, 1}|Results]}; - Subscribers -> - Count = lists:foldl(fun(Sub, Acc) -> - dispatch(Sub, Topic, Msg), Acc + 1 - end, 0, Subscribers), + SubPids -> + Count = lists:foldl(fun(SubPid, Acc) -> + dispatch(SubPid, Topic, Msg), Acc + 1 + end, 0, SubPids), Delivery#delivery{results = [{dispatch, Topic, Count}|Results]} end. -dispatch({SubPid, _SubId}, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; -dispatch({share, _Group, _Sub}, _Topic, _Msg) -> - ignored. +dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> + SubPid ! {dispatch, Topic, Msg}, + true; +%% TODO: how to optimize the share sub? +dispatch({share, _Group, _SubPid}, _Topic, _Msg) -> + false. inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; inc_dropped_cnt(_Topic) -> emqx_metrics:inc('messages/dropped'). --spec(subscribers(emqx_topic:topic()) -> [emqx_types:subscriber()]). +-spec(subscribers(emqx_topic:topic()) -> [pid()]). subscribers(Topic) -> - try ets:lookup_element(?SUBSCRIBER, Topic, 2) catch error:badarg -> [] end. + safe_lookup_element(?SUBSCRIBER, Topic, []). --spec(subscriptions(emqx_types:subscriber()) +%%------------------------------------------------------------------------------ +%% Subscriber is down +%%------------------------------------------------------------------------------ + +-spec(subscriber_down(pid()) -> true). +subscriber_down(SubPid) -> + lists:foreach( + fun(Sub = {_, Topic}) -> + case ets:lookup(?SUBOPTION, Sub) of + [{_, SubOpts}] -> + Group = maps:get(share, SubOpts, undefined), + true = ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), + true = ets:delete(?SUBOPTION, Sub), + gen_server:cast(pick(Topic), {unsubscribe, Group, Topic}); + [] -> ok + end + end, ets:lookup(?SUBSCRIPTION, SubPid)), + ets:delete(?SUBSCRIPTION, SubPid). + +%%------------------------------------------------------------------------------ +%% Management APIs +%%------------------------------------------------------------------------------ + +-spec(subscriptions(pid() | emqx_types:subid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(Subscriber) -> - lists:map(fun({_, {share, _Group, Topic}}) -> - subscription(Topic, Subscriber); - ({_, Topic}) -> - subscription(Topic, Subscriber) - end, ets:lookup(?SUBSCRIPTION, Subscriber)). +subscriptions(SubPid) -> + [{Topic, safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{})} + || Topic <- safe_lookup_element(?SUBSCRIPTION, SubPid, [])]. -subscription(Topic, Subscriber) -> - {Topic, ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)}. +-spec(subscribed(pid(), emqx_topic:topic()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + ets:member(?SUBOPTION, {SubPid, Topic}); +subscribed(SubId, Topic) when ?is_subid(SubId) -> + %%FIXME:... SubId -> SubPid + ets:member(?SUBOPTION, {SubId, Topic}). --spec(subscribed(emqx_topic:topic(), pid() | emqx_types:subid() | emqx_types:subscriber()) -> boolean()). -subscribed(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - case ets:match_object(?SUBOPTION, {{Topic, {SubPid, '_'}}, '_'}, 1) of - {Match, _} -> - length(Match) >= 1; - '$end_of_table' -> - false - end; -subscribed(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - case ets:match_object(?SUBOPTION, {{Topic, {'_', SubId}}, '_'}, 1) of - {Match, _} -> - length(Match) >= 1; - '$end_of_table' -> - false - end; -subscribed(Topic, {SubPid, SubId}) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - ets:member(?SUBOPTION, {Topic, {SubPid, SubId}}). +-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts()). +get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> + safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{}). --spec(get_subopts(emqx_topic:topic(), emqx_types:subscriber()) -> emqx_types:subopts()). -get_subopts(Topic, Subscriber) when is_binary(Topic) -> - try ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2) - catch error:badarg -> [] - end. - --spec(set_subopts(emqx_topic:topic(), emqx_types:subscriber(), emqx_types:subopts()) -> boolean()). -set_subopts(Topic, Subscriber, Opts) when is_binary(Topic), is_map(Opts) -> - case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of +-spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). +set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> + Sub = {self(), Topic}, + case ets:lookup(?SUBOPTION, Sub) of [{_, OldOpts}] -> - ets:insert(?SUBOPTION, {{Topic, Subscriber}, maps:merge(OldOpts, Opts)}); + ets:insert(?SUBOPTION, {Sub, maps:merge(OldOpts, NewOpts)}); [] -> false end. -async_call(Broker, Req) -> - From = {self(), Tag = make_ref()}, - ok = gen_server:cast(Broker, {From, Req}), - Tag. +-spec(topics() -> [emqx_topic:topic()]). +topics() -> + emqx_router:topics(). -wait_for_replies(Tags, Timeout) -> - lists:foreach( - fun(Tag) -> - wait_for_reply(Tag, Timeout) - end, Tags). +safe_lookup_element(Tab, Key, Def) -> + try ets:lookup_element(Tab, Key, 2) catch error:badarg -> Def end. -wait_for_reply(Tag, Timeout) -> - receive - {Tag, Reply} -> Reply - after Timeout -> - exit(timeout) +%%------------------------------------------------------------------------------ +%% Stats fun +%%------------------------------------------------------------------------------ + +stats_fun() -> + safe_update_stats(?SUBSCRIBER, 'subscribers/count', 'subscribers/max'), + safe_update_stats(?SUBSCRIPTION, 'subscriptions/count', 'subscriptions/max'), + safe_update_stats(?SUBOPTION, 'suboptions/count', 'suboptions/max'). + +safe_update_stats(Tab, Stat, MaxStat) -> + case ets:info(Tab, size) of + undefined -> ok; + Size -> emqx_stats:setstat(Stat, MaxStat, Size) end. -%% Pick a broker -pick(SubPid) when is_pid(SubPid) -> - gproc_pool:pick_worker(broker, SubPid). +%%------------------------------------------------------------------------------ +%% Pick and call +%%------------------------------------------------------------------------------ --spec(topics() -> [emqx_topic:topic()]). -topics() -> emqx_router:topics(). +call(Broker, Req) -> + gen_server:call(Broker, Req, ?TIMEOUT). + +%% Pick a broker +pick(Topic) -> + gproc_pool:pick_worker(broker, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks @@ -326,61 +321,32 @@ topics() -> emqx_router:topics(). init([Pool, Id]) -> true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, #state{pool = Pool, id = Id, submap = #{}, submon = emqx_pmon:new()}}. + {ok, #{pool => Pool, id => Id}}. + +handle_call({subscribe, Group, Topic}, _From, State) -> + Ok = emqx_router:add_route(Topic, dest(Group)), + {reply, Ok, State}; + +handle_call({unsubscribe, Group, Topic}, _From, State) -> + Ok = case ets:member(?SUBSCRIBER, Topic) of + false -> emqx_router:delete_route(Topic, dest(Group)); + true -> ok + end, + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({From, #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}}, State) -> - Subscriber = {SubPid, SubId}, - case ets:member(?SUBOPTION, {Topic, Subscriber}) of - false -> - resubscribe(From, {Subscriber, SubOpts, Topic}, State); - true -> - case ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2) =:= SubOpts of - true -> - gen_server:reply(From, ok), - {noreply, State}; - false -> - resubscribe(From, {Subscriber, SubOpts, Topic}, State) - end - end; - -handle_cast({From, #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}}, State) -> - Subscriber = {SubPid, SubId}, - case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of - [{_, SubOpts}] -> - Group = maps:get(share, SubOpts, undefined), - true = do_unsubscribe(Group, Topic, Subscriber), - emqx_shared_sub:unsubscribe(Group, Topic, SubPid), - case ets:member(?SUBSCRIBER, Topic) of - false -> emqx_router:del_route(From, Topic, dest(Group)); - true -> gen_server:reply(From, ok) - end; - [] -> gen_server:reply(From, ok) - end, - {noreply, State}; - handle_cast(Msg, State) -> emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #state{submap = SubMap}) -> - case maps:find(SubPid, SubMap) of - {ok, SubIds} -> - lists:foreach(fun(SubId) -> subscriber_down({SubPid, SubId}) end, SubIds), - {noreply, demonitor_subscriber(SubPid, State)}; - error -> - emqx_logger:error("unexpected 'DOWN': ~p, reason: ~p", [SubPid, Reason]), - {noreply, State} - end; - handle_info(Info, State) -> emqx_logger:error("[Broker] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{pool = Pool, id = Id}) -> +terminate(_Reason, #{pool := Pool, id := Id}) -> gproc_pool:disconnect_worker(Pool, {Pool, Id}). code_change(_OldVsn, State, _Extra) -> @@ -390,69 +356,9 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -resubscribe(From, {Subscriber, SubOpts, Topic}, State) -> - {SubPid, _} = Subscriber, - Group = maps:get(share, SubOpts, undefined), - true = do_subscribe(Group, Topic, Subscriber, SubOpts), - emqx_shared_sub:subscribe(Group, Topic, SubPid), - emqx_router:add_route(From, Topic, dest(Group)), - {noreply, monitor_subscriber(Subscriber, State)}. - -insert_subscriber(Group, Topic, Subscriber) -> - Subscribers = subscribers(Topic), - case lists:member(Subscriber, Subscribers) of - false -> - ets:insert(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}); - _ -> - ok - end. - -do_subscribe(Group, Topic, Subscriber, SubOpts) -> - ets:insert(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}), - insert_subscriber(Group, Topic, Subscriber), - ets:insert(?SUBOPTION, {{Topic, Subscriber}, SubOpts}). - -do_unsubscribe(Group, Topic, Subscriber) -> - ets:delete_object(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}), - ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}), - ets:delete(?SUBOPTION, {Topic, Subscriber}). - -subscriber_down(Subscriber) -> - Topics = lists:map(fun({_, {share, Group, Topic}}) -> - {Topic, Group}; - ({_, Topic}) -> - {Topic, undefined} - end, ets:lookup(?SUBSCRIPTION, Subscriber)), - lists:foreach(fun({Topic, undefined}) -> - true = do_unsubscribe(undefined, Topic, Subscriber), - ets:member(?SUBSCRIBER, Topic) orelse emqx_router:del_route(Topic, dest(undefined)); - ({Topic, Group}) -> - true = do_unsubscribe(Group, Topic, Subscriber), - Groups = groups(Topic), - case lists:member(Group, lists:usort(Groups)) of - true -> ok; - false -> emqx_router:del_route(Topic, dest(Group)) - end - end, Topics). - -monitor_subscriber({SubPid, SubId}, State = #state{submap = SubMap, submon = SubMon}) -> - UpFun = fun(SubIds) -> lists:usort([SubId|SubIds]) end, - State#state{submap = maps:update_with(SubPid, UpFun, [SubId], SubMap), - submon = emqx_pmon:monitor(SubPid, SubMon)}. - -demonitor_subscriber(SubPid, State = #state{submap = SubMap, submon = SubMon}) -> - State#state{submap = maps:remove(SubPid, SubMap), - submon = emqx_pmon:demonitor(SubPid, SubMon)}. - dest(undefined) -> node(); dest(Group) -> {Group, node()}. shared(undefined, Name) -> Name; shared(Group, Name) -> {share, Group, Name}. -groups(Topic) -> - lists:foldl(fun({_, {share, Group, _}}, Acc) -> - [Group | Acc]; - ({_, _}, Acc) -> - Acc - end, [], ets:lookup(?SUBSCRIBER, Topic)). diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index e597a233e..35fe06f0d 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -16,63 +16,82 @@ -behaviour(gen_server). --export([start_link/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-compile({no_auto_import, [monitor/2]}). -%% internal export --export([stats_fun/0]). +-export([start_link/0]). +-export([monitor/2]). +-export([create_seq/1, reclaim_seq/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). -define(HELPER, ?MODULE). +-define(SUBMON, emqx_submon). +-define(SUBSEQ, emqx_subseq). --record(state, {}). +-record(state, {pmon :: emqx_pmon:pmon()}). --spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). +-spec(monitor(pid(), emqx_types:subid()) -> ok). +monitor(SubPid, SubId) when is_pid(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [] -> + gen_server:cast(?HELPER, {monitor, SubPid, SubId}); + [{_, SubId}] -> + ok; + _Other -> + error(subid_conflict) + end. + +-spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +create_seq(Topic) -> + emqx_sequence:nextval(?SUBSEQ, Topic). + +-spec(reclaim_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +reclaim_seq(Topic) -> + emqx_sequence:reclaim(?SUBSEQ, Topic). + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ init([]) -> - %% Use M:F/A for callback, not anonymous function because - %% fun M:F/A is small, also no badfun risk during hot beam reload - emqx_stats:update_interval(broker_stats, fun ?MODULE:stats_fun/0), - {ok, #state{}, hibernate}. + %% SubSeq: Topic -> SeqId + _ = emqx_sequence:create(?SUBSEQ), + %% SubMon: SubPid -> SubId + _ = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), + %% Stats timer + emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), + {ok, #state{pmon = emqx_pmon:new()}, hibernate}. handle_call(Req, _From, State) -> emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]), {reply, ignored, State}. +handle_cast({monitor, SubPid, SubId}, State = #state{pmon = PMon}) -> + true = ets:insert(?SUBMON, {SubPid, SubId}), + {noreply, State#state{pmon = emqx_pmon:monitor(SubPid, PMon)}}; + handle_cast(Msg, State) -> emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), {noreply, State}. +handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMon}) -> + true = ets:delete(?SUBMON, SubPid), + ok = emqx_pool:async_submit(fun emqx_broker:subscriber_down/1, [SubPid]), + {noreply, State#state{pmon = emqx_pmon:erase(SubPid, PMon)}}; + handle_info(Info, State) -> emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, #state{}) -> + _ = emqx_sequence:delete(?SUBSEQ), emqx_stats:cancel_update(broker_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -stats_fun() -> - safe_update_stats(emqx_subscriber, - 'subscribers/count', 'subscribers/max'), - safe_update_stats(emqx_subscription, - 'subscriptions/count', 'subscriptions/max'), - safe_update_stats(emqx_suboptions, - 'suboptions/count', 'suboptions/max'). - -safe_update_stats(Tab, Stat, MaxStat) -> - case ets:info(Tab, size) of - undefined -> ok; - Size -> emqx_stats:setstat(Stat, MaxStat, Size) - end. - diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl index 51f6e72aa..a511e4154 100644 --- a/src/emqx_broker_sup.erl +++ b/src/emqx_broker_sup.erl @@ -20,8 +20,6 @@ -export([init/1]). --define(TAB_OPTS, [public, {read_concurrency, true}, {write_concurrency, true}]). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -30,39 +28,26 @@ start_link() -> %%------------------------------------------------------------------------------ init([]) -> - %% Create the pubsub tables - ok = lists:foreach(fun create_tab/1, [subscription, subscriber, suboption]), - + %% Broker pool + PoolSize = emqx_vm:schedulers() * 2, + BrokerPool = emqx_pool_sup:spec(emqx_broker_pool, + [broker, hash, PoolSize, + {emqx_broker, start_link, []}]), %% Shared subscription - SharedSub = {shared_sub, {emqx_shared_sub, start_link, []}, - permanent, 5000, worker, [emqx_shared_sub]}, + SharedSub = #{id => shared_sub, + start => {emqx_shared_sub, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_shared_sub]}, %% Broker helper - Helper = {broker_helper, {emqx_broker_helper, start_link, []}, - permanent, 5000, worker, [emqx_broker_helper]}, + Helper = #{id => helper, + start => {emqx_broker_helper, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_broker_helper]}, - %% Broker pool - BrokerPool = emqx_pool_sup:spec(emqx_broker_pool, - [broker, hash, emqx_vm:schedulers() * 2, - {emqx_broker, start_link, []}]), - - {ok, {{one_for_all, 0, 1}, [SharedSub, Helper, BrokerPool]}}. - -%%------------------------------------------------------------------------------ -%% Create tables -%%------------------------------------------------------------------------------ - -create_tab(suboption) -> - %% Suboption: {Topic, Sub} -> [{qos, 1}] - emqx_tables:new(emqx_suboption, [set | ?TAB_OPTS]); - -create_tab(subscriber) -> - %% Subscriber: Topic -> Sub1, Sub2, Sub3, ..., SubN - %% duplicate_bag: o(1) insert - emqx_tables:new(emqx_subscriber, [duplicate_bag | ?TAB_OPTS]); - -create_tab(subscription) -> - %% Subscription: Sub -> Topic1, Topic2, Topic3, ..., TopicN - %% bag: o(n) insert - emqx_tables:new(emqx_subscription, [bag | ?TAB_OPTS]). + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. diff --git a/src/emqx_local_bridge.erl b/src/emqx_local_bridge.erl index 7c4e7cea1..df2dda686 100644 --- a/src/emqx_local_bridge.erl +++ b/src/emqx_local_bridge.erl @@ -61,7 +61,7 @@ init([Pool, Id, Node, Topic, Options]) -> true -> true = erlang:monitor_node(Node, true), Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), - emqx_broker:subscribe(Topic, self(), #{share => Group, qos => ?QOS_0}), + emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}), State = parse_opts(Options, #state{node = Node, subtopic = Topic}), MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len, store_qos0 => true}), diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl index 62a882294..c4fc0fd08 100644 --- a/src/emqx_sequence.erl +++ b/src/emqx_sequence.erl @@ -14,45 +14,47 @@ -module(emqx_sequence). --export([create/0, create/1]). --export([generate/1, generate/2]). --export([reclaim/1, reclaim/2]). +-export([create/1, nextval/2, currval/2, reclaim/2, delete/1]). -type(key() :: term()). +-type(name() :: atom()). -type(seqid() :: non_neg_integer()). --define(DEFAULT_TAB, ?MODULE). +-export_type([seqid/0]). %% @doc Create a sequence. --spec(create() -> ok). -create() -> - create(?DEFAULT_TAB). - --spec(create(atom()) -> ok). -create(Tab) -> - _ = ets:new(Tab, [set, public, named_table, {write_concurrency, true}]), +-spec(create(name()) -> ok). +create(Name) -> + _ = ets:new(Name, [set, public, named_table, {write_concurrency, true}]), ok. -%% @doc Generate a sequence id. --spec(generate(key()) -> seqid()). -generate(Key) -> - generate(?DEFAULT_TAB, Key). +%% @doc Next value of the sequence. +-spec(nextval(name(), key()) -> seqid()). +nextval(Name, Key) -> + ets:update_counter(Name, Key, {2, 1}, {Key, 0}). --spec(generate(atom(), key()) -> seqid()). -generate(Tab, Key) -> - ets:update_counter(Tab, Key, {2, 1}, {Key, 0}). +%% @doc Current value of the sequence. +-spec(currval(name(), key()) -> seqid()). +currval(Name, Key) -> + try ets:lookup_element(Name, Key, 2) + catch + error:badarg -> 0 + end. %% @doc Reclaim a sequence id. --spec(reclaim(key()) -> seqid()). -reclaim(Key) -> - reclaim(?DEFAULT_TAB, Key). - --spec(reclaim(atom(), key()) -> seqid()). -reclaim(Tab, Key) -> - try ets:update_counter(Tab, Key, {2, -1, 0, 0}) of - 0 -> ets:delete_object(Tab, {Key, 0}), 0; +-spec(reclaim(name(), key()) -> seqid()). +reclaim(Name, Key) -> + try ets:update_counter(Name, Key, {2, -1, 0, 0}) of + 0 -> ets:delete_object(Name, {Key, 0}), 0; I -> I catch error:badarg -> 0 end. +%% @doc Delete the sequence. +delete(Name) -> + case ets:info(Name, name) of + Name -> ets:delete(Name); + undefined -> false + end. + diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 12718b9fc..262a9a7a8 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -465,7 +465,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), SubMap; {ok, _SubOpts} -> - emqx_broker:set_subopts(Topic, {self(), ClientId}, SubOpts), + emqx_broker:set_subopts(Topic, SubOpts), %% Why??? emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), maps:put(Topic, SubOpts, SubMap); diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl index 999a95723..f37b60d76 100644 --- a/test/emqx_sequence_SUITE.erl +++ b/test/emqx_sequence_SUITE.erl @@ -19,19 +19,19 @@ -include_lib("eunit/include/eunit.hrl"). --import(emqx_sequence, [generate/1, reclaim/1]). +-import(emqx_sequence, [nextval/2, reclaim/2]). all() -> [sequence_generate]. sequence_generate(_) -> - ok = emqx_sequence:create(), - ?assertEqual(1, generate(key)), - ?assertEqual(2, generate(key)), - ?assertEqual(3, generate(key)), - ?assertEqual(2, reclaim(key)), - ?assertEqual(1, reclaim(key)), - ?assertEqual(0, reclaim(key)), - ?assertEqual(false, ets:member(emqx_sequence, key)), - ?assertEqual(1, generate(key)). + ok = emqx_sequence:create(seqtab), + ?assertEqual(1, nextval(seqtab, key)), + ?assertEqual(2, nextval(seqtab, key)), + ?assertEqual(3, nextval(seqtab, key)), + ?assertEqual(2, reclaim(seqtab, key)), + ?assertEqual(1, reclaim(seqtab, key)), + ?assertEqual(0, reclaim(seqtab, key)), + ?assertEqual(false, ets:member(seqtab, key)), + ?assertEqual(1, nextval(seqtab, key)). From 36e7d63d66b47d9c6ba6f4de8d33a60b4ad0be31 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 7 Dec 2018 18:20:09 +0800 Subject: [PATCH 03/54] Implement subscription sharding. 1. Improve the design router, broker and shared_sub 2. New ets tables' design for subscription sharding --- src/emqx_access_control.erl | 2 +- src/emqx_acl_internal.erl | 2 +- src/emqx_broker.erl | 149 ++++++++++++++++++------------ src/emqx_broker_helper.erl | 14 ++- src/emqx_cm.erl | 6 +- src/emqx_ctl.erl | 2 +- src/emqx_hooks.erl | 2 +- src/emqx_metrics.erl | 2 +- src/emqx_router.erl | 175 +++++++++++++----------------------- src/emqx_sequence.erl | 3 +- src/emqx_shared_sub.erl | 63 ++++++++----- src/emqx_sm.erl | 8 +- src/emqx_stats.erl | 2 +- src/emqx_tables.erl | 6 +- src/emqx_zone.erl | 2 +- 15 files changed, 227 insertions(+), 211 deletions(-) diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 1b9d76937..06ea86633 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -148,7 +148,7 @@ stop() -> %%----------------------------------------------------------------------------- init([]) -> - _ = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]), {ok, #{}}. handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) -> diff --git a/src/emqx_acl_internal.erl b/src/emqx_acl_internal.erl index eee7e6c18..f8b995096 100644 --- a/src/emqx_acl_internal.erl +++ b/src/emqx_acl_internal.erl @@ -45,7 +45,7 @@ all_rules() -> -spec(init([File :: string()]) -> {ok, #{}}). init([File]) -> - _ = emqx_tables:new(?ACL_RULE_TAB, [set, public, {read_concurrency, true}]), + ok = emqx_tables:new(?ACL_RULE_TAB, [set, public, {read_concurrency, true}]), true = load_rules_from_file(File), {ok, #{acl_file => File}}. diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 3a3cde1fa..0a9264489 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -23,7 +23,7 @@ -export([unsubscribe/1, unsubscribe/2]). -export([subscriber_down/1]). -export([publish/1, safe_publish/1]). --export([dispatch/2, dispatch/3]). +-export([dispatch/2]). -export([subscriptions/1, subscribers/1, subscribed/2]). -export([get_subopts/2, set_subopts/2]). -export([topics/0]). @@ -34,8 +34,6 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --define(SHARD, 1024). --define(TIMEOUT, 60000). -define(BROKER, ?MODULE). %% ETS tables @@ -44,33 +42,36 @@ -define(SUBSCRIBER, emqx_subscriber). -define(SUBSCRIPTION, emqx_subscription). -%% Gards +%% Guards -define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> _ = create_tabs(), - gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], []). + Name = emqx_misc:proc_name(?BROKER, Id), + gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], []). %%------------------------------------------------------------------------------ %% Create tabs %%------------------------------------------------------------------------------ +-spec(create_tabs() -> ok). create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], - %% SubId: SubId -> SubPid1, SubPid2,... - _ = emqx_tables:new(?SUBID, [bag | TabOpts]), + %% SubId: SubId -> SubPid + ok = emqx_tables:new(?SUBID, [set | TabOpts]), + %% SubOption: {SubPid, Topic} -> SubOption - _ = emqx_tables:new(?SUBOPTION, [set | TabOpts]), + ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]), %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... %% duplicate_bag: o(1) insert - _ = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% duplicate_bag: o(1) insert - emqx_tables:new(?SUBSCRIBER, [duplicate_bag | TabOpts]). + ok = emqx_tables:new(?SUBSCRIBER, [duplicate_bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API @@ -92,14 +93,23 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> ok = emqx_broker_helper:monitor(SubPid, SubId), - Group = maps:get(share, SubOpts, undefined), %% true = ets:insert(?SUBID, {SubId, SubPid}), true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), - %% SeqId = emqx_broker_helper:create_seq(Topic), - true = ets:insert(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), - true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - ok = emqx_shared_sub:subscribe(Group, Topic, SubPid), - call(pick(Topic), {subscribe, Group, Topic}); + case maps:get(share, SubOpts, undefined) of + undefined -> + Shard = emqx_broker_helper:get_shard(SubPid, Topic), + case Shard of + 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}); + I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}) + end, + SubOpts1 = maps:put(shard, Shard, SubOpts), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts1}), + call(pick({Topic, Shard}), {subscribe, Topic}); + Group -> %% Shared subscription + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + emqx_shared_sub:subscribe(Group, Topic, SubPid) + end; true -> ok end. @@ -112,12 +122,21 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> - Group = maps:get(share, SubOpts, undefined), + _ = emqx_broker_helper:reclaim_seq(Topic), + case maps:get(share, SubOpts, undefined) of + undefined -> + case maps:get(shared, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + Group -> + ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid) + end, true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), - true = ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), - true = ets:delete(?SUBOPTION, {SubPid, Topic}), - ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid), - call(pick(Topic), {unsubscribe, Group, Topic}); + %%true = ets:delete_object(?SUBID, {SubId, SubPid}), + true = ets:delete(?SUBOPTION, {SubPid, Topic}); [] -> ok end. @@ -207,22 +226,23 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> emqx_hooks:run('message.dropped', [#{node => node()}, Msg]), inc_dropped_cnt(Topic), Delivery; - [SubPid] -> %% optimize? - dispatch(SubPid, Topic, Msg), + [Sub] -> %% optimize? + dispatch(Sub, Topic, Msg), Delivery#delivery{results = [{dispatch, Topic, 1}|Results]}; - SubPids -> - Count = lists:foldl(fun(SubPid, Acc) -> - dispatch(SubPid, Topic, Msg), Acc + 1 - end, 0, SubPids), + Subs -> + Count = lists:foldl( + fun(Sub, Acc) -> + dispatch(Sub, Topic, Msg), Acc + 1 + end, 0, Subs), Delivery#delivery{results = [{dispatch, Topic, Count}|Results]} end. dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}, - true; -%% TODO: how to optimize the share sub? -dispatch({share, _Group, _SubPid}, _Topic, _Msg) -> - false. + SubPid ! {dispatch, Topic, Msg}; +dispatch({shard, I}, Topic, Msg) -> + lists:foreach(fun(SubPid) -> + SubPid ! {dispatch, Topic, Msg} + end, safe_lookup_element(?SUBSCRIBER, {share, Topic, I}, [])). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; @@ -240,17 +260,20 @@ subscribers(Topic) -> -spec(subscriber_down(pid()) -> true). subscriber_down(SubPid) -> lists:foreach( - fun(Sub = {_, Topic}) -> + fun(Sub = {_Pid, Topic}) -> case ets:lookup(?SUBOPTION, Sub) of [{_, SubOpts}] -> - Group = maps:get(share, SubOpts, undefined), - true = ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, SubPid)}), - true = ets:delete(?SUBOPTION, Sub), - gen_server:cast(pick(Topic), {unsubscribe, Group, Topic}); + _ = emqx_broker_helper:reclaim_seq(Topic), + case maps:get(shared, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; [] -> ok end end, ets:lookup(?SUBSCRIPTION, SubPid)), - ets:delete(?SUBSCRIPTION, SubPid). + true = ets:delete(?SUBSCRIPTION, SubPid). %%------------------------------------------------------------------------------ %% Management APIs @@ -305,11 +328,14 @@ safe_update_stats(Tab, Stat, MaxStat) -> end. %%------------------------------------------------------------------------------ -%% Pick and call +%% call, cast, pick %%------------------------------------------------------------------------------ call(Broker, Req) -> - gen_server:call(Broker, Req, ?TIMEOUT). + gen_server:call(Broker, Req). + +cast(Broker, Msg) -> + gen_server:cast(Broker, Msg). %% Pick a broker pick(Topic) -> @@ -320,24 +346,41 @@ pick(Topic) -> %%------------------------------------------------------------------------------ init([Pool, Id]) -> + _ = emqx_router:set_mode(protected), true = gproc_pool:connect_worker(Pool, {Pool, Id}), {ok, #{pool => Pool, id => Id}}. -handle_call({subscribe, Group, Topic}, _From, State) -> - Ok = emqx_router:add_route(Topic, dest(Group)), - {reply, Ok, State}; - -handle_call({unsubscribe, Group, Topic}, _From, State) -> - Ok = case ets:member(?SUBSCRIBER, Topic) of - false -> emqx_router:delete_route(Topic, dest(Group)); - true -> ok - end, - {reply, Ok, State}; +handle_call({subscribe, Topic}, _From, State) -> + case get(Topic) of + undefined -> + _ = put(Topic, true), + emqx_router:add_route(Topic); + true -> ok + end, + {reply, ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), {reply, ignored, State}. +handle_cast({unsubscribed, Topic}, State) -> + case ets:member(?SUBSCRIBER, Topic) of + false -> + _ = erase(Topic), + emqx_router:delete_route(Topic); + true -> ok + end, + {noreply, State}; + +handle_cast({unsubscribed, Topic, I}, State) -> + case ets:member(?SUBSCRIBER, {shard, Topic, I}) of + false -> + true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {unsubscribed, Topic}); + true -> ok + end, + {noreply, State}; + handle_cast(Msg, State) -> emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]), {noreply, State}. @@ -356,9 +399,3 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -dest(undefined) -> node(); -dest(Group) -> {Group, node()}. - -shared(undefined, Name) -> Name; -shared(Group, Name) -> {share, Group, Name}. - diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index 35fe06f0d..d3e7f9d37 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -20,6 +20,7 @@ -export([start_link/0]). -export([monitor/2]). +-export([get_shard/2]). -export([create_seq/1, reclaim_seq/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, @@ -46,6 +47,13 @@ monitor(SubPid, SubId) when is_pid(SubPid) -> error(subid_conflict) end. +-spec(get_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_shard(SubPid, Topic) -> + case create_seq(Topic) of + Seq when Seq =< 1024 -> 0; + _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shards, 2)) + end. + -spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). create_seq(Topic) -> emqx_sequence:nextval(?SUBSEQ, Topic). @@ -60,9 +68,11 @@ reclaim_seq(Topic) -> init([]) -> %% SubSeq: Topic -> SeqId - _ = emqx_sequence:create(?SUBSEQ), + ok = emqx_sequence:create(?SUBSEQ), + %% Shards: CPU * 32 + true = ets:insert(?SUBSEQ, {shards, emqx_vm:schedulers() * 32}), %% SubMon: SubPid -> SubId - _ = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), + ok = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), %% Stats timer emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), {ok, #state{pmon = emqx_pmon:new()}, hibernate}. diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 19892b386..6756cf02b 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -125,9 +125,9 @@ notify(Msg) -> init([]) -> TabOpts = [public, set, {write_concurrency, true}], - _ = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), - _ = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), - _ = emqx_tables:new(?CONN_STATS_TAB, TabOpts), + ok = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?CONN_STATS_TAB, TabOpts), ok = emqx_stats:update_interval(cm_stats, fun ?MODULE:update_conn_stats/0), {ok, #{conn_pmon => emqx_pmon:new()}}. diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl index 17166a014..c00556eb7 100644 --- a/src/emqx_ctl.erl +++ b/src/emqx_ctl.erl @@ -96,7 +96,7 @@ usage() -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [ordered_set, protected]), + ok = emqx_tables:new(?TAB, [protected, ordered_set]), {ok, #state{seq = 0}}. handle_call(Req, _From, State) -> diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl index 073c12870..b2eb0d6f4 100644 --- a/src/emqx_hooks.erl +++ b/src/emqx_hooks.erl @@ -139,7 +139,7 @@ lookup(HookPoint) -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = Action}}, _From, State) -> diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index caf862146..b4b3a1307 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -285,7 +285,7 @@ qos_sent(?QOS_2) -> init([]) -> % Create metrics table - _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]), + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), lists:foreach(fun new/1, ?BYTES_METRICS ++ ?PACKET_METRICS ++ ?MESSAGE_METRICS), {ok, #{}, hibernate}. diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 941c004f7..313adc475 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -28,23 +28,22 @@ -export([start_link/2]). %% Route APIs --export([add_route/1, add_route/2, add_route/3]). +-export([add_route/1, add_route/2]). -export([get_routes/1]). --export([del_route/1, del_route/2, del_route/3]). +-export([delete_route/1, delete_route/2]). -export([has_routes/1, match_routes/1, print_routes/1]). -export([topics/0]). + +%% Mode +-export([set_mode/1, get_mode/0]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -type(destination() :: node() | {binary(), node()}). --record(batch, {enabled, timer, pending}). --record(state, {pool, id, batch :: #batch{}}). - -define(ROUTE, emqx_route). --define(BATCH(Enabled), #batch{enabled = Enabled}). --define(BATCH(Enabled, Pending), #batch{enabled = Enabled, pending = Pending}). %%------------------------------------------------------------------------------ %% Mnesia bootstrap @@ -62,49 +61,66 @@ mnesia(copy) -> ok = ekka_mnesia:copy_table(?ROUTE). %%------------------------------------------------------------------------------ -%% Strat a router +%% Start a router %%------------------------------------------------------------------------------ --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, - ?MODULE, [Pool, Id], [{hibernate_after, 2000}]). + Name = emqx_misc:proc_name(?MODULE, Id), + gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). %%------------------------------------------------------------------------------ %% Route APIs %%------------------------------------------------------------------------------ --spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok). +-spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). add_route(Topic) when is_binary(Topic) -> add_route(#route{topic = Topic, dest = node()}); add_route(Route = #route{topic = Topic}) -> - cast(pick(Topic), {add_route, Route}). + case get_mode() of + protected -> do_add_route(Route); + undefined -> call(pick(Topic), {add_route, Route}) + end. --spec(add_route(emqx_topic:topic(), destination()) -> ok). +-spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). add_route(Topic, Dest) when is_binary(Topic) -> add_route(#route{topic = Topic, dest = Dest}). --spec(add_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok). -add_route(From, Topic, Dest) when is_binary(Topic) -> - cast(pick(Topic), {add_route, From, #route{topic = Topic, dest = Dest}}). +%% @private +do_add_route(Route = #route{topic = Topic, dest = Dest}) -> + case lists:member(Route, get_routes(Topic)) of + true -> ok; + false -> + ok = emqx_router_helper:monitor(Dest), + case emqx_topic:wildcard(Topic) of + true -> trans(fun add_trie_route/1, [Route]); + false -> add_direct_route(Route) + end + end. -spec(get_routes(emqx_topic:topic()) -> [emqx_types:route()]). get_routes(Topic) -> ets:lookup(?ROUTE, Topic). --spec(del_route(emqx_topic:topic() | emqx_types:route()) -> ok). -del_route(Topic) when is_binary(Topic) -> - del_route(#route{topic = Topic, dest = node()}); -del_route(Route = #route{topic = Topic}) -> - cast(pick(Topic), {del_route, Route}). +-spec(delete_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). +delete_route(Topic) when is_binary(Topic) -> + delete_route(#route{topic = Topic, dest = node()}); +delete_route(Route = #route{topic = Topic}) -> + case get_mode() of + protected -> do_delete_route(Route); + undefined -> call(pick(Topic), {delete_route, Route}) + end. --spec(del_route(emqx_topic:topic(), destination()) -> ok). -del_route(Topic, Dest) when is_binary(Topic) -> - del_route(#route{topic = Topic, dest = Dest}). +-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +delete_route(Topic, Dest) when is_binary(Topic) -> + delete_route(#route{topic = Topic, dest = Dest}). --spec(del_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok). -del_route(From, Topic, Dest) when is_binary(Topic) -> - cast(pick(Topic), {del_route, From, #route{topic = Topic, dest = Dest}}). +%% @private +do_delete_route(Route = #route{topic = Topic}) -> + case emqx_topic:wildcard(Topic) of + true -> trans(fun del_trie_route/1, [Route]); + false -> del_direct_route(Route) + end. -spec(has_routes(emqx_topic:topic()) -> boolean()). has_routes(Topic) when is_binary(Topic) -> @@ -127,8 +143,15 @@ print_routes(Topic) -> io:format("~s -> ~s~n", [To, Dest]) end, match_routes(Topic)). -cast(Router, Msg) -> - gen_server:cast(Router, Msg). +-spec(set_mode(protected | atom()) -> any()). +set_mode(Mode) when is_atom(Mode) -> + put('$router_mode', Mode). + +-spec(get_mode() -> protected | undefined | atom()). +get_mode() -> get('$router_mode'). + +call(Router, Msg) -> + gen_server:call(Router, Msg, infinity). pick(Topic) -> gproc_pool:pick_worker(router, Topic). @@ -138,71 +161,28 @@ pick(Topic) -> %%------------------------------------------------------------------------------ init([Pool, Id]) -> - rand:seed(exsplus, erlang:timestamp()), - gproc_pool:connect_worker(Pool, {Pool, Id}), - Batch = #batch{enabled = emqx_config:get_env(route_batch_clean, false), - pending = sets:new()}, - {ok, ensure_batch_timer(#state{pool = Pool, id = Id, batch = Batch})}. + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +handle_call({add_route, Route}, _From, State) -> + {reply, do_add_route(Route), State}; + +handle_call({delete_route, Route}, _From, State) -> + {reply, do_delete_route(Route), State}; handle_call(Req, _From, State) -> emqx_logger:error("[Router] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({add_route, From, Route}, State) -> - {noreply, NewState} = handle_cast({add_route, Route}, State), - _ = gen_server:reply(From, ok), - {noreply, NewState}; - -handle_cast({add_route, Route = #route{topic = Topic, dest = Dest}}, State) -> - case lists:member(Route, get_routes(Topic)) of - true -> ok; - false -> - ok = emqx_router_helper:monitor(Dest), - case emqx_topic:wildcard(Topic) of - true -> log(trans(fun add_trie_route/1, [Route])); - false -> add_direct_route(Route) - end - end, - {noreply, State}; - -handle_cast({del_route, From, Route}, State) -> - {noreply, NewState} = handle_cast({del_route, Route}, State), - _ = gen_server:reply(From, ok), - {noreply, NewState}; - -handle_cast({del_route, Route = #route{topic = Topic, dest = Dest}}, State) when is_tuple(Dest) -> - {noreply, case emqx_topic:wildcard(Topic) of - true -> log(trans(fun del_trie_route/1, [Route])), - State; - false -> del_direct_route(Route, State) - end}; - -handle_cast({del_route, Route = #route{topic = Topic}}, State) -> - %% Confirm if there are still subscribers... - {noreply, case ets:member(emqx_subscriber, Topic) of - true -> State; - false -> - case emqx_topic:wildcard(Topic) of - true -> log(trans(fun del_trie_route/1, [Route])), - State; - false -> del_direct_route(Route, State) - end - end}; - handle_cast(Msg, State) -> emqx_logger:error("[Router] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({timeout, _TRef, batch_delete}, State = #state{batch = Batch}) -> - _ = del_direct_routes(sets:to_list(Batch#batch.pending)), - {noreply, ensure_batch_timer(State#state{batch = ?BATCH(true, sets:new())}), hibernate}; - handle_info(Info, State) -> emqx_logger:error("[Router] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{pool = Pool, id = Id, batch = Batch}) -> - _ = cacel_batch_timer(Batch), +terminate(_Reason, #{pool := Pool, id := Id}) -> gproc_pool:disconnect_worker(Pool, {Pool, Id}). code_change(_OldVsn, State, _Extra) -> @@ -212,17 +192,6 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -ensure_batch_timer(State = #state{batch = #batch{enabled = false}}) -> - State; -ensure_batch_timer(State = #state{batch = Batch}) -> - TRef = erlang:start_timer(50 + rand:uniform(50), self(), batch_delete), - State#state{batch = Batch#batch{timer = TRef}}. - -cacel_batch_timer(#batch{enabled = false}) -> - ok; -cacel_batch_timer(#batch{enabled = true, timer = TRef}) -> - catch erlang:cancel_timer(TRef). - add_direct_route(Route) -> mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]). @@ -233,25 +202,9 @@ add_trie_route(Route = #route{topic = Topic}) -> end, mnesia:write(?ROUTE, Route, sticky_write). -del_direct_route(Route, State = #state{batch = ?BATCH(false)}) -> - del_direct_route(Route), State; -del_direct_route(Route, State = #state{batch = Batch = ?BATCH(true, Pending)}) -> - State#state{batch = Batch#batch{pending = sets:add_element(Route, Pending)}}. - del_direct_route(Route) -> mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]). -del_direct_routes([]) -> - ok; -del_direct_routes(Routes) -> - DelFun = fun(R = #route{topic = Topic}) -> - case ets:member(emqx_subscriber, Topic) of - true -> ok; - false -> mnesia:delete_object(?ROUTE, R, sticky_write) - end - end, - mnesia:async_dirty(fun lists:foreach/2, [DelFun, Routes]). - del_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [Route] -> %% Remove route and trie @@ -270,7 +223,3 @@ trans(Fun, Args) -> {aborted, Error} -> {error, Error} end. -log(ok) -> ok; -log({error, Reason}) -> - emqx_logger:error("[Router] mnesia aborted: ~p", [Reason]). - diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl index c4fc0fd08..022531df5 100644 --- a/src/emqx_sequence.erl +++ b/src/emqx_sequence.erl @@ -25,8 +25,7 @@ %% @doc Create a sequence. -spec(create(name()) -> ok). create(Name) -> - _ = ets:new(Name, [set, public, named_table, {write_concurrency, true}]), - ok. + emqx_tables:new(Name, [public, set, {write_concurrency, true}]). %% @doc Next value of the sequence. -spec(nextval(name(), key()) -> seqid()). diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index ebe6d51f8..d1d0d921d 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -17,6 +17,7 @@ -behaviour(gen_server). -include("emqx.hrl"). +-include("emqx_mqtt.hrl"). %% Mnesia bootstrap -export([mnesia/1]). @@ -27,7 +28,8 @@ -export([start_link/0]). -export([subscribe/3, unsubscribe/3]). --export([dispatch/3, maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]). +-export([dispatch/3]). +-export([maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]). %% for testing -export([subscribers/2]). @@ -38,6 +40,7 @@ -define(SERVER, ?MODULE). -define(TAB, emqx_shared_subscription). +-define(SHARED_SUBS, emqx_shared_subscriber). -define(ALIVE_SUBS, emqx_alive_shared_subscribers). -define(SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS, 5). -define(ack, shared_sub_ack). @@ -48,8 +51,6 @@ -record(state, {pmon}). -record(emqx_shared_subscription, {group, topic, subpid}). --include("emqx_mqtt.hrl"). - %%------------------------------------------------------------------------------ %% Mnesia bootstrap %%------------------------------------------------------------------------------ @@ -72,16 +73,11 @@ mnesia(copy) -> start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). -subscribe(undefined, _Topic, _SubPid) -> - ok; subscribe(Group, Topic, SubPid) when is_pid(SubPid) -> - mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), - gen_server:cast(?SERVER, {monitor, SubPid}). + gen_server:call(?SERVER, {subscribe, Group, Topic, SubPid}). -unsubscribe(undefined, _Topic, _SubPid) -> - ok; unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) -> - mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)). + gen_server:call(?SERVER, {unsubscribe, Group, Topic, SubPid}). record(Group, Topic, SubPid) -> #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}. @@ -251,14 +247,16 @@ do_pick_subscriber(Group, Topic, round_robin, _ClientId, Count) -> subscribers(Group, Topic) -> ets:select(?TAB, [{{emqx_shared_subscription, Group, Topic, '$1'}, [], ['$1']}]). -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> - {atomic, PMon} = mnesia:transaction(fun init_monitors/0), + _ = emqx_router:set_mode(protected), mnesia:subscribe({table, ?TAB, simple}), - ets:new(?ALIVE_SUBS, [named_table, {read_concurrency, true}, protected]), + {atomic, PMon} = mnesia:transaction(fun init_monitors/0), + ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> @@ -267,14 +265,29 @@ init_monitors() -> emqx_pmon:monitor(SubPid, Mon) end, emqx_pmon:new(), ?TAB). +handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) -> + mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:add_route(Topic, {Group, node()}) + end, + ok = maybe_insert_alive_tab(SubPid), + true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), + {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; + +handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> + mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + end, + {reply, ok, State}; + handle_call(Req, _From, State) -> emqx_logger:error("[SharedSub] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({monitor, SubPid}, State= #state{pmon = PMon}) -> - NewPmon = emqx_pmon:monitor(SubPid, PMon), - ok = maybe_insert_alive_tab(SubPid), - {noreply, update_stats(State#state{pmon = NewPmon})}; handle_cast(Msg, State) -> emqx_logger:error("[SharedSub] unexpected cast: ~p", [Msg]), {noreply, State}. @@ -316,12 +329,18 @@ maybe_insert_alive_tab(Pid) when is_pid(Pid) -> ets:insert(?ALIVE_SUBS, {Pid}), cleanup_down(SubPid) -> ?IS_LOCAL_PID(SubPid) orelse ets:delete(?ALIVE_SUBS, SubPid), lists:foreach( - fun(Record) -> - mnesia:dirty_delete_object(?TAB, Record) - end,mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). + fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) -> + ok = mnesia:dirty_delete_object(?TAB, Record), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + end + end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). update_stats(State) -> - emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), State. + emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), + State. %% Return 'true' if the subscriber process is alive AND not in the failed list is_active_sub(Pid, FailedSubs) -> diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 8f9a3e3cb..d178a8ae7 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -202,10 +202,10 @@ notify(Event) -> init([]) -> TabOpts = [public, set, {write_concurrency, true}], - _ = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]), - _ = emqx_tables:new(?SESSION_P_TAB, TabOpts), - _ = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), - _ = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), {ok, #{session_pmon => emqx_pmon:new()}}. diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index 61ff6cbc3..790c397b9 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -152,7 +152,7 @@ cast(Msg) -> %%------------------------------------------------------------------------------ init(#{tick_ms := TickMs}) -> - _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]), + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([?CONNECTION_STATS, ?SESSION_STATS, ?PUBSUB_STATS, ?ROUTE_STATS, ?RETAINED_STATS]), true = ets:insert(?TAB, [{Name, 0} || Name <- Stats]), diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl index 330c87d9c..9b3ebfeae 100644 --- a/src/emqx_tables.erl +++ b/src/emqx_tables.erl @@ -17,10 +17,12 @@ -export([new/2]). %% Create a named_table ets. +-spec(new(atom(), list()) -> ok). new(Tab, Opts) -> case ets:info(Tab, name) of undefined -> - ets:new(Tab, lists:usort([named_table | Opts])); - Tab -> Tab + _ = ets:new(Tab, lists:usort([named_table | Opts])), + ok; + Tab -> ok end. diff --git a/src/emqx_zone.erl b/src/emqx_zone.erl index dd183dbdf..d119abe52 100644 --- a/src/emqx_zone.erl +++ b/src/emqx_zone.erl @@ -68,7 +68,7 @@ stop() -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [set, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [set, {read_concurrency, true}]), {ok, element(2, handle_info(reload, #{timer => undefined}))}. handle_call(force_reload, _From, State) -> From 5e53eaeee5b65d2d1ec0c0721d5cdddc311527f4 Mon Sep 17 00:00:00 2001 From: turtled Date: Sat, 8 Dec 2018 09:56:00 +0800 Subject: [PATCH 04/54] rename shard shared --- src/emqx_broker.erl | 29 ++++++++++++++++------------- src/emqx_broker_helper.erl | 12 ++++++------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 0a9264489..380e16c42 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -71,7 +71,7 @@ create_tabs() -> %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% duplicate_bag: o(1) insert - ok = emqx_tables:new(?SUBSCRIBER, [duplicate_bag | TabOpts]). + ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API @@ -97,15 +97,16 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), case maps:get(share, SubOpts, undefined) of undefined -> - Shard = emqx_broker_helper:get_shard(SubPid, Topic), - case Shard of + Shared = emqx_broker_helper:get_shared(SubPid, Topic), + case Shared of 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}); - I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}) + I -> + true = ets:insert(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), + true = ets:insert(?SUBSCRIBER, {Topic, {shared, I}}) end, - SubOpts1 = maps:put(shard, Shard, SubOpts), + SubOpts1 = maps:put(shared, Shared, SubOpts), true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts1}), - call(pick({Topic, Shard}), {subscribe, Topic}); + call(pick({Topic, Shared}), {subscribe, Topic}); Group -> %% Shared subscription true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), emqx_shared_sub:subscribe(Group, Topic, SubPid) @@ -128,7 +129,7 @@ unsubscribe(Topic) when is_binary(Topic) -> case maps:get(shared, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + I -> true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; Group -> @@ -239,10 +240,11 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> SubPid ! {dispatch, Topic, Msg}; -dispatch({shard, I}, Topic, Msg) -> +dispatch({shared, I}, Topic, Msg) -> + lists:foreach(fun(SubPid) -> SubPid ! {dispatch, Topic, Msg} - end, safe_lookup_element(?SUBSCRIBER, {share, Topic, I}, [])). + end, safe_lookup_element(?SUBSCRIBER, {shared, Topic, I}, [])). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; @@ -267,7 +269,8 @@ subscriber_down(SubPid) -> case maps:get(shared, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + I -> true = ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}), + true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; [] -> ok @@ -373,9 +376,9 @@ handle_cast({unsubscribed, Topic}, State) -> {noreply, State}; handle_cast({unsubscribed, Topic, I}, State) -> - case ets:member(?SUBSCRIBER, {shard, Topic, I}) of + case ets:member(?SUBSCRIBER, {shared, Topic, I}) of false -> - true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), + true = ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}), cast(pick(Topic), {unsubscribed, Topic}); true -> ok end, diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index d3e7f9d37..6830b4d32 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -20,7 +20,7 @@ -export([start_link/0]). -export([monitor/2]). --export([get_shard/2]). +-export([get_shared/2]). -export([create_seq/1, reclaim_seq/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, @@ -47,11 +47,11 @@ monitor(SubPid, SubId) when is_pid(SubPid) -> error(subid_conflict) end. --spec(get_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). -get_shard(SubPid, Topic) -> +-spec(get_shared(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_shared(SubPid, Topic) -> case create_seq(Topic) of Seq when Seq =< 1024 -> 0; - _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shards, 2)) + _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shareds, 2)) end. -spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). @@ -69,8 +69,8 @@ reclaim_seq(Topic) -> init([]) -> %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), - %% Shards: CPU * 32 - true = ets:insert(?SUBSEQ, {shards, emqx_vm:schedulers() * 32}), + %% Shareds: CPU * 32 + true = ets:insert(?SUBSEQ, {shareds, emqx_vm:schedulers() * 32}), %% SubMon: SubPid -> SubId ok = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), %% Stats timer From ba897e51f9f27de998ba638c69d65c33b8578ce1 Mon Sep 17 00:00:00 2001 From: turtled Date: Sat, 8 Dec 2018 10:26:50 +0800 Subject: [PATCH 05/54] Subscriber down clear emqx_suboption table --- src/emqx_broker.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 380e16c42..5f3e63059 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -272,7 +272,8 @@ subscriber_down(SubPid) -> I -> true = ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}), true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; + end, + ets:delete(?SUBOPTION, Sub); [] -> ok end end, ets:lookup(?SUBSCRIPTION, SubPid)), From d1be51d398afdf46133baeff0b13320b92721e93 Mon Sep 17 00:00:00 2001 From: turtled Date: Sat, 8 Dec 2018 10:52:15 +0800 Subject: [PATCH 06/54] Format code --- src/emqx_broker.erl | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 5f3e63059..b9332ffed 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -127,10 +127,16 @@ unsubscribe(Topic) when is_binary(Topic) -> case maps:get(share, SubOpts, undefined) of undefined -> case maps:get(shared, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + 0 -> + true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> + true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), + case ets:member(emqx_subscriber, {shared, Topic, I}) of + true -> ok; + false -> ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}) + end, + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; Group -> ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid) @@ -267,11 +273,16 @@ subscriber_down(SubPid) -> [{_, SubOpts}] -> _ = emqx_broker_helper:reclaim_seq(Topic), case maps:get(shared, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}), - true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + 0 -> + true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> + true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), + case ets:member(emqx_subscriber, {shared, Topic, I}) of + true -> ok; + false -> ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}) + end, + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end, ets:delete(?SUBOPTION, Sub); [] -> ok From 5164d0d6a57e3be187d3d5dfee1b605dbd4f688f Mon Sep 17 00:00:00 2001 From: turtled Date: Sat, 8 Dec 2018 11:40:08 +0800 Subject: [PATCH 07/54] Fix unsubscribe fail and rename shared -> shard --- src/emqx_broker.erl | 43 +++++++++++++++++++------------------- src/emqx_broker_helper.erl | 12 +++++------ 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index b9332ffed..6556a59e5 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -97,19 +97,19 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), case maps:get(share, SubOpts, undefined) of undefined -> - Shared = emqx_broker_helper:get_shared(SubPid, Topic), - case Shared of + Shard = emqx_broker_helper:get_shard(SubPid, Topic), + case Shard of 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}); I -> - true = ets:insert(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), - true = ets:insert(?SUBSCRIBER, {Topic, {shared, I}}) + true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}) end, - SubOpts1 = maps:put(shared, Shared, SubOpts), + SubOpts1 = maps:put(shard, Shard, SubOpts), true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts1}), - call(pick({Topic, Shared}), {subscribe, Topic}); - Group -> %% Shared subscription + call(pick({Topic, Shard}), {subscribe, Topic}); + Group -> %% Shard subscription true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - emqx_shared_sub:subscribe(Group, Topic, SubPid) + emqx_shard_sub:subscribe(Group, Topic, SubPid) end; true -> ok end. @@ -126,15 +126,15 @@ unsubscribe(Topic) when is_binary(Topic) -> _ = emqx_broker_helper:reclaim_seq(Topic), case maps:get(share, SubOpts, undefined) of undefined -> - case maps:get(shared, SubOpts, 0) of + case maps:get(shard, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), ok = cast(pick(Topic), {unsubscribed, Topic}); I -> - true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shared, Topic, I}) of + true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + case ets:member(emqx_subscriber, {shard, Topic, I}) of true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}) + false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) end, ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; @@ -143,7 +143,8 @@ unsubscribe(Topic) when is_binary(Topic) -> end, true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), %%true = ets:delete_object(?SUBID, {SubId, SubPid}), - true = ets:delete(?SUBOPTION, {SubPid, Topic}); + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + ok; [] -> ok end. @@ -246,11 +247,11 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> SubPid ! {dispatch, Topic, Msg}; -dispatch({shared, I}, Topic, Msg) -> +dispatch({shard, I}, Topic, Msg) -> lists:foreach(fun(SubPid) -> SubPid ! {dispatch, Topic, Msg} - end, safe_lookup_element(?SUBSCRIBER, {shared, Topic, I}, [])). + end, safe_lookup_element(?SUBSCRIBER, {shard, Topic, I}, [])). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; @@ -272,15 +273,15 @@ subscriber_down(SubPid) -> case ets:lookup(?SUBOPTION, Sub) of [{_, SubOpts}] -> _ = emqx_broker_helper:reclaim_seq(Topic), - case maps:get(shared, SubOpts, 0) of + case maps:get(shard, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), ok = cast(pick(Topic), {unsubscribed, Topic}); I -> - true = ets:delete_object(?SUBSCRIBER, {{shared, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shared, Topic, I}) of + true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + case ets:member(emqx_subscriber, {shard, Topic, I}) of true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}) + false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) end, ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) end, @@ -388,9 +389,9 @@ handle_cast({unsubscribed, Topic}, State) -> {noreply, State}; handle_cast({unsubscribed, Topic, I}, State) -> - case ets:member(?SUBSCRIBER, {shared, Topic, I}) of + case ets:member(?SUBSCRIBER, {shard, Topic, I}) of false -> - true = ets:delete_object(?SUBSCRIBER, {Topic, {shared, I}}), + true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), cast(pick(Topic), {unsubscribed, Topic}); true -> ok end, diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index 6830b4d32..d3e7f9d37 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -20,7 +20,7 @@ -export([start_link/0]). -export([monitor/2]). --export([get_shared/2]). +-export([get_shard/2]). -export([create_seq/1, reclaim_seq/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, @@ -47,11 +47,11 @@ monitor(SubPid, SubId) when is_pid(SubPid) -> error(subid_conflict) end. --spec(get_shared(pid(), emqx_topic:topic()) -> non_neg_integer()). -get_shared(SubPid, Topic) -> +-spec(get_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_shard(SubPid, Topic) -> case create_seq(Topic) of Seq when Seq =< 1024 -> 0; - _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shareds, 2)) + _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shards, 2)) end. -spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). @@ -69,8 +69,8 @@ reclaim_seq(Topic) -> init([]) -> %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), - %% Shareds: CPU * 32 - true = ets:insert(?SUBSEQ, {shareds, emqx_vm:schedulers() * 32}), + %% Shards: CPU * 32 + true = ets:insert(?SUBSEQ, {shards, emqx_vm:schedulers() * 32}), %% SubMon: SubPid -> SubId ok = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), %% Stats timer From b6c123b173ec2bb88830c3f9f1045b9bf44d7bf3 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Fri, 30 Nov 2018 10:22:18 +0100 Subject: [PATCH 08/54] Use git tag for app vsn --- Makefile | 1 - erlang.mk | 2 +- src/emqx.app.src | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e815e0756..695cc6871 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ PROJECT = emqx PROJECT_DESCRIPTION = EMQ X Broker -PROJECT_VERSION = 3.0.0 DEPS = jsx gproc gen_rpc ekka esockd cowboy clique diff --git a/erlang.mk b/erlang.mk index f38d22653..c5d4b4f7f 100644 --- a/erlang.mk +++ b/erlang.mk @@ -1186,7 +1186,7 @@ else fi $(appsrc_verbose) cat src/$(PROJECT).app.src \ | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ - | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ + | sed "s/{vsn,[[:space:]]*\"git\"}/{vsn, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ > ebin/$(PROJECT).app endif diff --git a/src/emqx.app.src b/src/emqx.app.src index 1bca0118b..ce643634e 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,6 +1,6 @@ {application,emqx, [{description,"EMQ X Broker"}, - {vsn,"3.0.0"}, + {vsn,"git"}, {modules,[]}, {registered,[emqx_sup]}, {applications,[kernel,stdlib,jsx,gproc,gen_rpc,esockd, From 10288827d1c4292308b226c2abcff14e65e40396 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Fri, 30 Nov 2018 11:02:02 +0100 Subject: [PATCH 09/54] Fallback to git clone -n then checkout if git version is older than 1.8 --- Makefile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 695cc6871..16435bdab 100644 --- a/Makefile +++ b/Makefile @@ -47,17 +47,19 @@ PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets pu DIALYZER_DIRS := ebin/ DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns -GIT_VSN = $(shell git --version | grep -oE "[0-9]{1,2}\.[0-9]{1,2}") -GIT_VSN_17_COMP = $(shell echo -e "$(GIT_VSN)\n1.7" | sort -V | tail -1) -ifeq ($(GIT_VSN_17_COMP),1.7) - MAYBE_SHALLOW = -else - MAYBE_SHALLOW = -c advice.detachedHead=false --depth 1 -endif +GIT_VSN := $(shell git --version | grep -oE "[0-9]{1,2}\.[0-9]{1,2}") +GIT_VSN_17_COMP := $(shell echo -e "$(GIT_VSN)\n1.7" | sort -V | tail -1) +ifeq ($(GIT_VSN_17_COMP),1.7) define dep_fetch_git-emqx - git clone $(MAYBE_SHALLOW) -q -b $(call dep_commit,$(1)) -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)) > /dev/null 2>&1 + git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ + cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1)) endef +else +define dep_fetch_git-emqx + git clone -q -c advice.detachedHead=false --depth 1 -b $(call dep_commit,$(1)) -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)) +endef +endif core_http_get-emqx = curl -Lf$(if $(filter-out 0,$(V)),,s)o $(call core_native_path,$1) $2 From 0b70896456867787a12fa5bd897bdc9b15f44ad1 Mon Sep 17 00:00:00 2001 From: Gilbert Date: Fri, 30 Nov 2018 19:16:48 +0800 Subject: [PATCH 10/54] Fix the coverage shaky (#2010) --- src/emqx_broker.erl | 5 +++++ test/emqx_broker_SUITE.erl | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index b7e0e457e..ca4a86c87 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -33,6 +33,11 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + -record(state, {pool, id, submap, submon}). -record(subscribe, {topic, subpid, subid, subopts = #{}}). -record(unsubscribe, {topic, subpid, subid}). diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 3187aafa4..b9bfb4257 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -38,6 +38,7 @@ groups() -> publish, pubsub, t_local_subscribe, t_shared_subscribe, + dispatch_with_no_sub, 'pubsub#', 'pubsub+']}, {session, [sequence], [start_session]}, {metrics, [sequence], [inc_dec_metric]}, @@ -76,6 +77,11 @@ publish(_) -> emqx:publish(Msg), ?assert(receive {dispatch, <<"test/+">>, Msg} -> true after 5 -> false end). +dispatch_with_no_sub(_) -> + Msg = emqx_message:make(ct, <<"no_subscribers">>, <<"hello">>), + Delivery = #delivery{sender = self(), message = Msg, results = []}, + ?assertEqual(Delivery, emqx_broker:route([{<<"no_subscribers">>, node()}], Delivery)). + pubsub(_) -> true = emqx:is_running(node()), Self = self(), @@ -193,4 +199,3 @@ set_alarms(_) -> ?assertEqual(1, length(Alarms)), emqx_alarm_mgr:clear_alarm(<<"1">>), [] = emqx_alarm_mgr:get_alarms(). - From 3712d0c90fe251cea0b7bb1e173152e49c741f9f Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Sun, 2 Dec 2018 16:15:45 +0800 Subject: [PATCH 11/54] Add eunit tests to increase coverage. --- test/emqx_reason_codes_tests.erl | 130 +++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 test/emqx_reason_codes_tests.erl diff --git a/test/emqx_reason_codes_tests.erl b/test/emqx_reason_codes_tests.erl new file mode 100644 index 000000000..29f7ad081 --- /dev/null +++ b/test/emqx_reason_codes_tests.erl @@ -0,0 +1,130 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License") +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_reason_codes_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx_mqtt.hrl"). + +-import(lists, [seq/2, zip/2, foreach/2]). + +-define(MQTTV4_CODE_NAMES, [connection_acceptd, + unacceptable_protocol_version, + client_identifier_not_valid, + server_unavaliable, + malformed_username_or_password, + unauthorized_client, + unknown_error]). + +-define(MQTTV5_CODE_NAMES, [success, granted_qos1, granted_qos2, disconnect_with_will_message, + no_matching_subscribers, no_subscription_existed, continue_authentication, + re_authenticate, unspecified_error, malformed_Packet, protocol_error, + implementation_specific_error, unsupported_protocol_version, + client_identifier_not_valid, bad_username_or_password, not_authorized, + server_unavailable, server_busy, banned,server_shutting_down, + bad_authentication_method, keepalive_timeout, session_taken_over, + topic_filter_invalid, topic_name_invalid, packet_identifier_inuse, + packet_identifier_not_found, receive_maximum_exceeded, topic_alias_invalid, + packet_too_large, message_rate_too_high, quota_exceeded, + administrative_action, payload_format_invalid, retain_not_supported, + qos_not_supported, use_another_server, server_moved, + shared_subscriptions_not_supported, connection_rate_exceeded, + maximum_connect_time, subscription_identifiers_not_supported, + wildcard_subscriptions_not_supported, unknown_error]). + +-define(MQTTV5_CODES, [16#00, 16#01, 16#02, 16#04, 16#10, 16#11, 16#18, 16#19, 16#80, 16#81, 16#82, + 16#83, 16#84, 16#85, 16#86, 16#87, 16#88, 16#89, 16#8A, 16#8B, 16#8C, 16#8D, + 16#8E, 16#8F, 16#90, 16#91, 16#92, 16#93, 16#94, 16#95, 16#96, 16#97, 16#98, + 16#99, 16#9A, 16#9B, 16#9C, 16#9D, 16#9E, 16#9F, 16#A0, 16#A1, 16#A2, code]). + +-define(MQTTV5_TXT, [<<"Success">>, <<"Granted QoS 1">>, <<"Granted QoS 2">>, + <<"Disconnect with Will Message">>, <<"No matching subscribers">>, + <<"No subscription existed">>, <<"Continue authentication">>, + <<"Re-authenticate">>, <<"Unspecified error">>, <<"Malformed Packet">>, + <<"Protocol Error">>, <<"Implementation specific error">>, + <<"Unsupported Protocol Version">>, <<"Client Identifier not valid">>, + <<"Bad User Name or Password">>, <<"Not authorized">>, + <<"Server unavailable">>, <<"Server busy">>, <<"Banned">>, + <<"Server shutting down">>, <<"Bad authentication method">>, + <<"Keep Alive timeout">>, <<"Session taken over">>, + <<"Topic Filter invalid">>, <<"Topic Name invalid">>, + <<"Packet Identifier in use">>, <<"Packet Identifier not found">>, + <<"Receive Maximum exceeded">>, <<"Topic Alias invalid">>, + <<"Packet too large">>, <<"Message rate too high">>, <<"Quota exceeded">>, + <<"Administrative action">>, <<"Payload format invalid">>, + <<"Retain not supported">>, <<"QoS not supported">>, + <<"Use another server">>, <<"Server moved">>, + <<"Shared Subscriptions not supported">>, <<"Connection rate exceeded">>, + <<"Maximum connect time">>, <<"Subscription Identifiers not supported">>, + <<"Wildcard Subscriptions not supported">>, <<"Unknown error">>]). + +-define(COMPAT_CODES_V5, [16#80, 16#81, 16#82, 16#83, 16#84, 16#85, 16#86, 16#87, + 16#88, 16#89, 16#8A, 16#8B, 16#8C, 16#97, 16#9C, 16#9D, + 16#9F]). + +-define(COMPAT_CODES_V4, [?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, + ?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, + ?CONNACK_INVALID_ID, + ?CONNACK_CREDENTIALS, + ?CONNACK_AUTH, + ?CONNACK_SERVER, + ?CONNACK_SERVER, + ?CONNACK_AUTH, + ?CONNACK_SERVER, + ?CONNACK_AUTH, + ?CONNACK_SERVER, ?CONNACK_SERVER, ?CONNACK_SERVER, ?CONNACK_SERVER]). + +mqttv4_name_test() -> + (((codes_test(?MQTT_PROTO_V4)) + (seq(0,6))) + (?MQTTV4_CODE_NAMES)) + (fun emqx_reason_codes:name/2). + +mqttv5_name_test() -> + (((codes_test(?MQTT_PROTO_V5)) + (?MQTTV5_CODES)) + (?MQTTV5_CODE_NAMES)) + (fun emqx_reason_codes:name/2). + +text_test() -> + (((codes_test(?MQTT_PROTO_V5)) + (?MQTTV5_CODES)) + (?MQTTV5_TXT)) + (fun emqx_reason_codes:text/1). + +compat_test() -> + (((codes_test(connack)) + (?COMPAT_CODES_V5)) + (?COMPAT_CODES_V4)) + (fun emqx_reason_codes:compat/2), + (((codes_test(suback)) + ([0,1,2, 16#80])) + ([0,1,2, 16#80])) + (fun emqx_reason_codes:compat/2). + +codes_test(AsistVar) -> + fun(CODES) -> + fun(NAMES) -> + fun(Procedure) -> + foreach(fun({Code, Result}) -> + ?assertEqual(Result, case erlang:fun_info(Procedure, name) of + {name, text} -> Procedure(Code); + {name, name} -> Procedure(Code, AsistVar); + {name, compat} -> Procedure(AsistVar, Code) + end) + end, zip(CODES, NAMES)) + end + end + end. From f008ceb5c82b3a6948dd0f585e8af1492bfd1de2 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 30 Nov 2018 20:37:10 +0800 Subject: [PATCH 12/54] Optimize the route and trie modules. 1. Use mnesia:wread/1 to replace mnesia:read/2 2. Update the router supervisor --- src/emqx_router.erl | 15 ++++++++------- src/emqx_router_helper.erl | 4 ++-- src/emqx_router_sup.erl | 8 ++++++-- src/emqx_trie.erl | 14 +++++++------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 2ac656d56..941c004f7 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -34,7 +34,8 @@ -export([has_routes/1, match_routes/1, print_routes/1]). -export([topics/0]). %% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). -type(destination() :: node() | {binary(), node()}). @@ -45,9 +46,9 @@ -define(BATCH(Enabled), #batch{enabled = Enabled}). -define(BATCH(Enabled, Pending), #batch{enabled = Enabled, pending = Pending}). -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Mnesia bootstrap -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ mnesia(boot) -> ok = ekka_mnesia:create_table(?ROUTE, [ @@ -132,9 +133,9 @@ cast(Router, Msg) -> pick(Topic) -> gproc_pool:pick_worker(router, Topic). -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([Pool, Id]) -> rand:seed(exsplus, erlang:timestamp()), @@ -207,9 +208,9 @@ terminate(_Reason, #state{pool = Pool, id = Id, batch = Batch}) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Internal functions -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ ensure_batch_timer(State = #state{batch = #batch{enabled = false}}) -> State; diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index 7bacddd4c..c24b10715 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -84,8 +84,8 @@ monitor(Node) when is_atom(Node) -> %%------------------------------------------------------------------------------ init([]) -> - ekka:monitor(membership), - mnesia:subscribe({table, ?ROUTING_NODE, simple}), + _ = ekka:monitor(membership), + _ = mnesia:subscribe({table, ?ROUTING_NODE, simple}), Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of diff --git a/src/emqx_router_sup.erl b/src/emqx_router_sup.erl index 004b88bb8..2bbaabc18 100644 --- a/src/emqx_router_sup.erl +++ b/src/emqx_router_sup.erl @@ -24,8 +24,12 @@ start_link() -> init([]) -> %% Router helper - Helper = {router_helper, {emqx_router_helper, start_link, []}, - permanent, 5000, worker, [emqx_router_helper]}, + Helper = #{id => helper, + start => {emqx_router_helper, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_router_helper]}, %% Router pool RouterPool = emqx_pool_sup:spec(emqx_router_pool, diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 569de9092..79f6042b7 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -62,10 +62,10 @@ mnesia(copy) -> %% Trie APIs %%------------------------------------------------------------------------------ -%% @doc Insert a topic into the trie +%% @doc Insert a topic filter into the trie. -spec(insert(emqx_topic:topic()) -> ok). insert(Topic) when is_binary(Topic) -> - case mnesia:read(?TRIE_NODE, Topic) of + case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{topic = Topic}] -> ok; [TrieNode = #trie_node{topic = undefined}] -> @@ -77,21 +77,21 @@ insert(Topic) when is_binary(Topic) -> write_trie_node(#trie_node{node_id = Topic, topic = Topic}) end. -%% @doc Find trie nodes that match the topic +%% @doc Find trie nodes that match the topic name. -spec(match(emqx_topic:topic()) -> list(emqx_topic:topic())). match(Topic) when is_binary(Topic) -> TrieNodes = match_node(root, emqx_topic:words(Topic)), [Name || #trie_node{topic = Name} <- TrieNodes, Name =/= undefined]. -%% @doc Lookup a trie node +%% @doc Lookup a trie node. -spec(lookup(NodeId :: binary()) -> [#trie_node{}]). lookup(NodeId) -> mnesia:read(?TRIE_NODE, NodeId). -%% @doc Delete a topic from the trie +%% @doc Delete a topic filter from the trie. -spec(delete(emqx_topic:topic()) -> ok). delete(Topic) when is_binary(Topic) -> - case mnesia:read(?TRIE_NODE, Topic) of + case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{edge_count = 0}] -> mnesia:delete({?TRIE_NODE, Topic}), delete_path(lists:reverse(emqx_topic:triples(Topic))); @@ -108,7 +108,7 @@ delete(Topic) when is_binary(Topic) -> %% @doc Add a path to the trie. add_path({Node, Word, Child}) -> Edge = #trie_edge{node_id = Node, word = Word}, - case mnesia:read(?TRIE_NODE, Node) of + case mnesia:wread({?TRIE_NODE, Node}) of [TrieNode = #trie_node{edge_count = Count}] -> case mnesia:wread({?TRIE, Edge}) of [] -> From a32b043980f85f030010db863ce6ef28a47fb80f Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sat, 1 Dec 2018 19:50:17 +0100 Subject: [PATCH 13/54] Download erlang.mk and use git tag in appfile vsn --- .gitignore | 1 + Makefile | 24 +- erlang.mk | 2741 ---------------------------------------------------- 3 files changed, 2 insertions(+), 2764 deletions(-) delete mode 100644 erlang.mk diff --git a/.gitignore b/.gitignore index 80e3bf42d..7a4e891d1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ compile_commands.json cuttlefish rebar.lock xrefr +erlang.mk diff --git a/Makefile b/Makefile index 16435bdab..26bcf22ce 100644 --- a/Makefile +++ b/Makefile @@ -47,29 +47,7 @@ PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets pu DIALYZER_DIRS := ebin/ DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns -GIT_VSN := $(shell git --version | grep -oE "[0-9]{1,2}\.[0-9]{1,2}") -GIT_VSN_17_COMP := $(shell echo -e "$(GIT_VSN)\n1.7" | sort -V | tail -1) - -ifeq ($(GIT_VSN_17_COMP),1.7) -define dep_fetch_git-emqx - git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ - cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1)) -endef -else -define dep_fetch_git-emqx - git clone -q -c advice.detachedHead=false --depth 1 -b $(call dep_commit,$(1)) -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)) -endef -endif - -core_http_get-emqx = curl -Lf$(if $(filter-out 0,$(V)),,s)o $(call core_native_path,$1) $2 - -define dep_fetch_hex-emqx - mkdir -p $(ERLANG_MK_TMP)/hex $(DEPS_DIR)/$1; \ - $(call core_http_get-emqx,$(ERLANG_MK_TMP)/hex/$1.tar,\ - https://repo.hex.pm/tarballs/$1-$(strip $(word 2,$(dep_$1))).tar); \ - tar -xOf $(ERLANG_MK_TMP)/hex/$1.tar contents.tar.gz | tar -C $(DEPS_DIR)/$1 -xzf -; -endef - +$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk) include erlang.mk clean:: gen-clean diff --git a/erlang.mk b/erlang.mk deleted file mode 100644 index c5d4b4f7f..000000000 --- a/erlang.mk +++ /dev/null @@ -1,2741 +0,0 @@ -# Copyright (c) 2013-2015, Loïc Hoguin -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -.PHONY: all app apps deps search rel docs install-docs check tests clean distclean help erlang-mk - -ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST))) - -ERLANG_MK_VERSION = 2.0.0-pre.2-130-gc6fe5ea - -# Core configuration. - -PROJECT ?= $(notdir $(CURDIR)) -PROJECT := $(strip $(PROJECT)) - -PROJECT_VERSION ?= rolling -PROJECT_MOD ?= $(PROJECT)_app - -# Verbosity. - -V ?= 0 - -verbose_0 = @ -verbose_2 = set -x; -verbose = $(verbose_$(V)) - -gen_verbose_0 = @echo " GEN " $@; -gen_verbose_2 = set -x; -gen_verbose = $(gen_verbose_$(V)) - -# Temporary files directory. - -ERLANG_MK_TMP ?= $(CURDIR)/.erlang.mk -export ERLANG_MK_TMP - -# "erl" command. - -ERL = erl +A0 -noinput -boot start_clean - -# Platform detection. - -ifeq ($(PLATFORM),) -UNAME_S := $(shell uname -s) - -ifeq ($(UNAME_S),Linux) -PLATFORM = linux -else ifeq ($(UNAME_S),Darwin) -PLATFORM = darwin -else ifeq ($(UNAME_S),SunOS) -PLATFORM = solaris -else ifeq ($(UNAME_S),GNU) -PLATFORM = gnu -else ifeq ($(UNAME_S),FreeBSD) -PLATFORM = freebsd -else ifeq ($(UNAME_S),NetBSD) -PLATFORM = netbsd -else ifeq ($(UNAME_S),OpenBSD) -PLATFORM = openbsd -else ifeq ($(UNAME_S),DragonFly) -PLATFORM = dragonfly -else ifeq ($(shell uname -o),Msys) -PLATFORM = msys2 -else -$(error Unable to detect platform. Please open a ticket with the output of uname -a.) -endif - -export PLATFORM -endif - -# Core targets. - -all:: deps app rel - -# Noop to avoid a Make warning when there's nothing to do. -rel:: - $(verbose) : - -check:: tests - -clean:: clean-crashdump - -clean-crashdump: -ifneq ($(wildcard erl_crash.dump),) - $(gen_verbose) rm -f erl_crash.dump -endif - -distclean:: clean distclean-tmp - -distclean-tmp: - $(gen_verbose) rm -rf $(ERLANG_MK_TMP) - -help:: - $(verbose) printf "%s\n" \ - "erlang.mk (version $(ERLANG_MK_VERSION)) is distributed under the terms of the ISC License." \ - "Copyright (c) 2013-2015 Loïc Hoguin " \ - "" \ - "Usage: [V=1] $(MAKE) [target]..." \ - "" \ - "Core targets:" \ - " all Run deps, app and rel targets in that order" \ - " app Compile the project" \ - " deps Fetch dependencies (if needed) and compile them" \ - " search q=... Search for a package in the built-in index" \ - " rel Build a release for this project, if applicable" \ - " docs Build the documentation for this project" \ - " install-docs Install the man pages for this project" \ - " check Compile and run all tests and analysis for this project" \ - " tests Run the tests for this project" \ - " clean Delete temporary and output files from most targets" \ - " distclean Delete all temporary and output files" \ - " help Display this help and exit" \ - " erlang-mk Update erlang.mk to the latest version" - -# Core functions. - -empty := -space := $(empty) $(empty) -tab := $(empty) $(empty) -comma := , - -define newline - - -endef - -define comma_list -$(subst $(space),$(comma),$(strip $(1))) -endef - -# Adding erlang.mk to make Erlang scripts who call init:get_plain_arguments() happy. -define erlang -$(ERL) $(2) -pz $(ERLANG_MK_TMP)/rebar/ebin -eval "$(subst $(newline),,$(subst ",\",$(1)))" -- erlang.mk -endef - -ifeq ($(PLATFORM),msys2) -core_native_path = $(subst \,\\\\,$(shell cygpath -w $1)) -else -core_native_path = $1 -endif - -ifeq ($(shell which wget 2>/dev/null | wc -l), 1) -define core_http_get - wget --no-check-certificate -O $(1) $(2)|| rm $(1) -endef -else -define core_http_get.erl - ssl:start(), - inets:start(), - case httpc:request(get, {"$(2)", []}, [{autoredirect, true}], []) of - {ok, {{_, 200, _}, _, Body}} -> - case file:write_file("$(1)", Body) of - ok -> ok; - {error, R1} -> halt(R1) - end; - {error, R2} -> - halt(R2) - end, - halt(0). -endef - -define core_http_get - $(call erlang,$(call core_http_get.erl,$(call core_native_path,$1),$2)) -endef -endif - -core_eq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1))) - -core_find = $(if $(wildcard $1),$(shell find $(1:%/=%) -type f -name $(subst *,\*,$2))) - -core_lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$(1))))))))))))))))))))))))))) - -core_ls = $(filter-out $(1),$(shell echo $(1))) - -# @todo Use a solution that does not require using perl. -core_relpath = $(shell perl -e 'use File::Spec; print File::Spec->abs2rel(@ARGV) . "\n"' $1 $2) - -# Automated update. - -ERLANG_MK_REPO ?= https://github.com/ninenines/erlang.mk -ERLANG_MK_COMMIT ?= -ERLANG_MK_BUILD_CONFIG ?= build.config -ERLANG_MK_BUILD_DIR ?= .erlang.mk.build - -erlang-mk: - git clone $(ERLANG_MK_REPO) $(ERLANG_MK_BUILD_DIR) -ifdef ERLANG_MK_COMMIT - cd $(ERLANG_MK_BUILD_DIR) && git checkout $(ERLANG_MK_COMMIT) -endif - if [ -f $(ERLANG_MK_BUILD_CONFIG) ]; then cp $(ERLANG_MK_BUILD_CONFIG) $(ERLANG_MK_BUILD_DIR)/build.config; fi - $(MAKE) -C $(ERLANG_MK_BUILD_DIR) - cp $(ERLANG_MK_BUILD_DIR)/erlang.mk ./erlang.mk - rm -rf $(ERLANG_MK_BUILD_DIR) - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: search - -define pkg_print - $(verbose) printf "%s\n" \ - $(if $(call core_eq,$(1),$(pkg_$(1)_name)),,"Pkg name: $(1)") \ - "App name: $(pkg_$(1)_name)" \ - "Description: $(pkg_$(1)_description)" \ - "Home page: $(pkg_$(1)_homepage)" \ - "Fetch with: $(pkg_$(1)_fetch)" \ - "Repository: $(pkg_$(1)_repo)" \ - "Commit: $(pkg_$(1)_commit)" \ - "" - -endef - -search: -ifdef q - $(foreach p,$(PACKAGES), \ - $(if $(findstring $(call core_lc,$(q)),$(call core_lc,$(pkg_$(p)_name) $(pkg_$(p)_description))), \ - $(call pkg_print,$(p)))) -else - $(foreach p,$(PACKAGES),$(call pkg_print,$(p))) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-deps - -# Configuration. - -ifdef OTP_DEPS -$(warning The variable OTP_DEPS is deprecated in favor of LOCAL_DEPS.) -endif - -IGNORE_DEPS ?= -export IGNORE_DEPS - -APPS_DIR ?= $(CURDIR)/apps -export APPS_DIR - -DEPS_DIR ?= $(CURDIR)/deps -export DEPS_DIR - -REBAR_DEPS_DIR = $(DEPS_DIR) -export REBAR_DEPS_DIR - -dep_name = $(if $(dep_$(1)),$(1),$(if $(pkg_$(1)_name),$(pkg_$(1)_name),$(1))) -dep_repo = $(patsubst git://github.com/%,https://github.com/%, \ - $(if $(dep_$(1)),$(word 2,$(dep_$(1))),$(pkg_$(1)_repo))) -dep_commit = $(if $(dep_$(1)_commit),$(dep_$(1)_commit),$(if $(dep_$(1)),$(word 3,$(dep_$(1))),$(pkg_$(1)_commit))) - -ALL_APPS_DIRS = $(if $(wildcard $(APPS_DIR)/),$(filter-out $(APPS_DIR),$(shell find $(APPS_DIR) -maxdepth 1 -type d))) -ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call dep_name,$(dep)))) - -ifeq ($(filter $(APPS_DIR) $(DEPS_DIR),$(subst :, ,$(ERL_LIBS))),) -ifeq ($(ERL_LIBS),) - ERL_LIBS = $(APPS_DIR):$(DEPS_DIR) -else - ERL_LIBS := $(ERL_LIBS):$(APPS_DIR):$(DEPS_DIR) -endif -endif -export ERL_LIBS - -export NO_AUTOPATCH - -# Verbosity. - -dep_verbose_0 = @echo " DEP " $(1); -dep_verbose_2 = set -x; -dep_verbose = $(dep_verbose_$(V)) - -# Core targets. - -ifdef IS_APP -apps:: -else -apps:: $(ALL_APPS_DIRS) -ifeq ($(IS_APP)$(IS_DEP),) - $(verbose) rm -f $(ERLANG_MK_TMP)/apps.log -endif - $(verbose) mkdir -p $(ERLANG_MK_TMP) -# Create ebin directory for all apps to make sure Erlang recognizes them -# as proper OTP applications when using -include_lib. This is a temporary -# fix, a proper fix would be to compile apps/* in the right order. - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - mkdir -p $$dep/ebin || exit $$?; \ - done - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/apps.log; then \ - :; \ - else \ - echo $$dep >> $(ERLANG_MK_TMP)/apps.log; \ - $(MAKE) -C $$dep IS_APP=1 || exit $$?; \ - fi \ - done -endif - -ifneq ($(SKIP_DEPS),) -deps:: -else -deps:: $(ALL_DEPS_DIRS) apps -ifeq ($(IS_APP)$(IS_DEP),) - $(verbose) rm -f $(ERLANG_MK_TMP)/deps.log -endif - $(verbose) mkdir -p $(ERLANG_MK_TMP) - $(verbose) for dep in $(ALL_DEPS_DIRS) ; do \ - if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/deps.log; then \ - :; \ - else \ - echo $$dep >> $(ERLANG_MK_TMP)/deps.log; \ - if [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \ - $(MAKE) -C $$dep IS_DEP=1 || exit $$?; \ - else \ - echo "Error: No Makefile to build dependency $$dep."; \ - exit 2; \ - fi \ - fi \ - done -endif - -# Deps related targets. - -# @todo rename GNUmakefile and makefile into Makefile first, if they exist -# While Makefile file could be GNUmakefile or makefile, -# in practice only Makefile is needed so far. -define dep_autopatch - if [ -f $(DEPS_DIR)/$(1)/erlang.mk ]; then \ - $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ - $(call dep_autopatch_erlang_mk,$(1)); \ - elif [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ - if [ 0 != `grep -c "include ../\w*\.mk" $(DEPS_DIR)/$(1)/Makefile` ]; then \ - $(call dep_autopatch2,$(1)); \ - elif [ 0 != `grep -ci rebar $(DEPS_DIR)/$(1)/Makefile` ]; then \ - $(call dep_autopatch2,$(1)); \ - elif [ -n "`find $(DEPS_DIR)/$(1)/ -type f -name \*.mk -not -name erlang.mk -exec grep -i rebar '{}' \;`" ]; then \ - $(call dep_autopatch2,$(1)); \ - else \ - $(call erlang,$(call dep_autopatch_app.erl,$(1))); \ - fi \ - else \ - if [ ! -d $(DEPS_DIR)/$(1)/src/ ]; then \ - $(call dep_autopatch_noop,$(1)); \ - else \ - $(call dep_autopatch2,$(1)); \ - fi \ - fi -endef - -define dep_autopatch2 - if [ -f $(DEPS_DIR)/$1/src/$1.app.src.script ]; then \ - $(call erlang,$(call dep_autopatch_appsrc_script.erl,$(1))); \ - fi; \ - $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ - if [ -f $(DEPS_DIR)/$(1)/rebar -o -f $(DEPS_DIR)/$(1)/rebar.config -o -f $(DEPS_DIR)/$(1)/rebar.config.script ]; then \ - $(call dep_autopatch_fetch_rebar); \ - $(call dep_autopatch_rebar,$(1)); \ - else \ - $(call dep_autopatch_gen,$(1)); \ - fi -endef - -define dep_autopatch_noop - printf "noop:\n" > $(DEPS_DIR)/$(1)/Makefile -endef - -# Overwrite erlang.mk with the current file by default. -ifeq ($(NO_AUTOPATCH_ERLANG_MK),) -define dep_autopatch_erlang_mk - echo "include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk" \ - > $(DEPS_DIR)/$1/erlang.mk -endef -else -define dep_autopatch_erlang_mk - : -endef -endif - -define dep_autopatch_gen - printf "%s\n" \ - "ERLC_OPTS = +debug_info" \ - "include ../../erlang.mk" > $(DEPS_DIR)/$(1)/Makefile -endef - -define dep_autopatch_fetch_rebar - mkdir -p $(ERLANG_MK_TMP); \ - if [ ! -d $(ERLANG_MK_TMP)/rebar ]; then \ - git clone -q -n -- https://github.com/rebar/rebar $(ERLANG_MK_TMP)/rebar; \ - cd $(ERLANG_MK_TMP)/rebar; \ - git checkout -q 791db716b5a3a7671e0b351f95ddf24b848ee173; \ - $(MAKE); \ - cd -; \ - fi -endef - -define dep_autopatch_rebar - if [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ - mv $(DEPS_DIR)/$(1)/Makefile $(DEPS_DIR)/$(1)/Makefile.orig.mk; \ - fi; \ - $(call erlang,$(call dep_autopatch_rebar.erl,$(1))); \ - rm -f $(DEPS_DIR)/$(1)/ebin/$(1).app -endef - -define dep_autopatch_rebar.erl - application:load(rebar), - application:set_env(rebar, log_level, debug), - Conf1 = case file:consult("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config)") of - {ok, Conf0} -> Conf0; - _ -> [] - end, - {Conf, OsEnv} = fun() -> - case filelib:is_file("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)") of - false -> {Conf1, []}; - true -> - Bindings0 = erl_eval:new_bindings(), - Bindings1 = erl_eval:add_binding('CONFIG', Conf1, Bindings0), - Bindings = erl_eval:add_binding('SCRIPT', "$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings1), - Before = os:getenv(), - {ok, Conf2} = file:script("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings), - {Conf2, lists:foldl(fun(E, Acc) -> lists:delete(E, Acc) end, os:getenv(), Before)} - end - end(), - Write = fun (Text) -> - file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/Makefile)", Text, [append]) - end, - Escape = fun (Text) -> - re:replace(Text, "\\\\$$", "\$$$$", [global, {return, list}]) - end, - Write("IGNORE_DEPS += edown eper eunit_formatters meck node_package " - "rebar_lock_deps_plugin rebar_vsn_plugin reltool_util\n"), - Write("C_SRC_DIR = /path/do/not/exist\n"), - Write("C_SRC_TYPE = rebar\n"), - Write("DRV_CFLAGS = -fPIC\nexport DRV_CFLAGS\n"), - Write(["ERLANG_ARCH = ", rebar_utils:wordsize(), "\nexport ERLANG_ARCH\n"]), - fun() -> - Write("ERLC_OPTS = +debug_info\nexport ERLC_OPTS\n"), - case lists:keyfind(erl_opts, 1, Conf) of - false -> ok; - {_, ErlOpts} -> - lists:foreach(fun - ({d, D}) -> - Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); - ({i, I}) -> - Write(["ERLC_OPTS += -I ", I, "\n"]); - ({platform_define, Regex, D}) -> - case rebar_utils:is_arch(Regex) of - true -> Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); - false -> ok - end; - ({parse_transform, PT}) -> - Write("ERLC_OPTS += +'{parse_transform, " ++ atom_to_list(PT) ++ "}'\n"); - (_) -> ok - end, ErlOpts) - end, - Write("\n") - end(), - fun() -> - File = case lists:keyfind(deps, 1, Conf) of - false -> []; - {_, Deps} -> - [begin case case Dep of - {N, S} when is_atom(N), is_list(S) -> {N, {hex, S}}; - {N, S} when is_tuple(S) -> {N, S}; - {N, _, S} -> {N, S}; - {N, _, S, _} -> {N, S}; - _ -> false - end of - false -> ok; - {Name, Source} -> - {Method, Repo, Commit} = case Source of - {hex, V} -> {hex, V, undefined}; - {git, R} -> {git, R, master}; - {M, R, {branch, C}} -> {M, R, C}; - {M, R, {ref, C}} -> {M, R, C}; - {M, R, {tag, C}} -> {M, R, C}; - {M, R, C} -> {M, R, C} - end, - Write(io_lib:format("DEPS += ~s\ndep_~s = ~s ~s ~s~n", [Name, Name, Method, Repo, Commit])) - end end || Dep <- Deps] - end - end(), - fun() -> - case lists:keyfind(erl_first_files, 1, Conf) of - false -> ok; - {_, Files} -> - Names = [[" ", case lists:reverse(F) of - "lre." ++ Elif -> lists:reverse(Elif); - Elif -> lists:reverse(Elif) - end] || "src/" ++ F <- Files], - Write(io_lib:format("COMPILE_FIRST +=~s\n", [Names])) - end - end(), - Write("\n\nrebar_dep: preprocess pre-deps deps pre-app app\n"), - Write("\npreprocess::\n"), - Write("\npre-deps::\n"), - Write("\npre-app::\n"), - PatchHook = fun(Cmd) -> - case Cmd of - "make -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); - "gmake -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); - "make " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); - "gmake " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); - _ -> Escape(Cmd) - end - end, - fun() -> - case lists:keyfind(pre_hooks, 1, Conf) of - false -> ok; - {_, Hooks} -> - [case H of - {'get-deps', Cmd} -> - Write("\npre-deps::\n\t" ++ PatchHook(Cmd) ++ "\n"); - {compile, Cmd} -> - Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); - {Regex, compile, Cmd} -> - case rebar_utils:is_arch(Regex) of - true -> Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); - false -> ok - end; - _ -> ok - end || H <- Hooks] - end - end(), - ShellToMk = fun(V) -> - re:replace(re:replace(V, "(\\\\$$)(\\\\w*)", "\\\\1(\\\\2)", [global]), - "-Werror\\\\b", "", [{return, list}, global]) - end, - PortSpecs = fun() -> - case lists:keyfind(port_specs, 1, Conf) of - false -> - case filelib:is_dir("$(call core_native_path,$(DEPS_DIR)/$1/c_src)") of - false -> []; - true -> - [{"priv/" ++ proplists:get_value(so_name, Conf, "$(1)_drv.so"), - proplists:get_value(port_sources, Conf, ["c_src/*.c"]), []}] - end; - {_, Specs} -> - lists:flatten([case S of - {Output, Input} -> {ShellToMk(Output), Input, []}; - {Regex, Output, Input} -> - case rebar_utils:is_arch(Regex) of - true -> {ShellToMk(Output), Input, []}; - false -> [] - end; - {Regex, Output, Input, [{env, Env}]} -> - case rebar_utils:is_arch(Regex) of - true -> {ShellToMk(Output), Input, Env}; - false -> [] - end - end || S <- Specs]) - end - end(), - PortSpecWrite = fun (Text) -> - file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/c_src/Makefile.erlang.mk)", Text, [append]) - end, - case PortSpecs of - [] -> ok; - _ -> - Write("\npre-app::\n\t$$\(MAKE) -f c_src/Makefile.erlang.mk\n"), - PortSpecWrite(io_lib:format("ERL_CFLAGS = -finline-functions -Wall -fPIC -I \\"~s/erts-~s/include\\" -I \\"~s\\"\n", - [code:root_dir(), erlang:system_info(version), code:lib_dir(erl_interface, include)])), - PortSpecWrite(io_lib:format("ERL_LDFLAGS = -L \\"~s\\" -lerl_interface -lei\n", - [code:lib_dir(erl_interface, lib)])), - [PortSpecWrite(["\n", E, "\n"]) || E <- OsEnv], - FilterEnv = fun(Env) -> - lists:flatten([case E of - {_, _} -> E; - {Regex, K, V} -> - case rebar_utils:is_arch(Regex) of - true -> {K, V}; - false -> [] - end - end || E <- Env]) - end, - MergeEnv = fun(Env) -> - lists:foldl(fun ({K, V}, Acc) -> - case lists:keyfind(K, 1, Acc) of - false -> [{K, rebar_utils:expand_env_variable(V, K, "")}|Acc]; - {_, V0} -> [{K, rebar_utils:expand_env_variable(V, K, V0)}|Acc] - end - end, [], Env) - end, - PortEnv = case lists:keyfind(port_env, 1, Conf) of - false -> []; - {_, PortEnv0} -> FilterEnv(PortEnv0) - end, - PortSpec = fun ({Output, Input0, Env}) -> - filelib:ensure_dir("$(call core_native_path,$(DEPS_DIR)/$1/)" ++ Output), - Input = [[" ", I] || I <- Input0], - PortSpecWrite([ - [["\n", K, " = ", ShellToMk(V)] || {K, V} <- lists:reverse(MergeEnv(PortEnv))], - case $(PLATFORM) of - darwin -> "\n\nLDFLAGS += -flat_namespace -undefined suppress"; - _ -> "" - end, - "\n\nall:: ", Output, "\n\n", - "%.o: %.c\n\t$$\(CC) -c -o $$\@ $$\< $$\(CFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.C\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.cc\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.cpp\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - [[Output, ": ", K, " = ", ShellToMk(V), "\n"] || {K, V} <- lists:reverse(MergeEnv(FilterEnv(Env)))], - Output, ": $$\(foreach ext,.c .C .cc .cpp,", - "$$\(patsubst %$$\(ext),%.o,$$\(filter %$$\(ext),$$\(wildcard", Input, "))))\n", - "\t$$\(CC) -o $$\@ $$\? $$\(LDFLAGS) $$\(ERL_LDFLAGS) $$\(DRV_LDFLAGS) $$\(EXE_LDFLAGS)", - case {filename:extension(Output), $(PLATFORM)} of - {[], _} -> "\n"; - {_, darwin} -> "\n"; - _ -> " -shared\n" - end]) - end, - [PortSpec(S) || S <- PortSpecs] - end, - Write("\ninclude $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk"), - RunPlugin = fun(Plugin, Step) -> - case erlang:function_exported(Plugin, Step, 2) of - false -> ok; - true -> - c:cd("$(call core_native_path,$(DEPS_DIR)/$1/)"), - Ret = Plugin:Step({config, "", Conf, dict:new(), dict:new(), dict:new(), - dict:store(base_dir, "", dict:new())}, undefined), - io:format("rebar plugin ~p step ~p ret ~p~n", [Plugin, Step, Ret]) - end - end, - fun() -> - case lists:keyfind(plugins, 1, Conf) of - false -> ok; - {_, Plugins} -> - [begin - case lists:keyfind(deps, 1, Conf) of - false -> ok; - {_, Deps} -> - case lists:keyfind(P, 1, Deps) of - false -> ok; - _ -> - Path = "$(call core_native_path,$(DEPS_DIR)/)" ++ atom_to_list(P), - io:format("~s", [os:cmd("$(MAKE) -C $(call core_native_path,$(DEPS_DIR)/$1) " ++ Path)]), - io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1")]), - code:add_patha(Path ++ "/ebin") - end - end - end || P <- Plugins], - [case code:load_file(P) of - {module, P} -> ok; - _ -> - case lists:keyfind(plugin_dir, 1, Conf) of - false -> ok; - {_, PluginsDir} -> - ErlFile = "$(call core_native_path,$(DEPS_DIR)/$1/)" ++ PluginsDir ++ "/" ++ atom_to_list(P) ++ ".erl", - {ok, P, Bin} = compile:file(ErlFile, [binary]), - {module, P} = code:load_binary(P, ErlFile, Bin) - end - end || P <- Plugins], - [RunPlugin(P, preprocess) || P <- Plugins], - [RunPlugin(P, pre_compile) || P <- Plugins], - [RunPlugin(P, compile) || P <- Plugins] - end - end(), - halt() -endef - -define dep_autopatch_app.erl - UpdateModules = fun(App) -> - case filelib:is_regular(App) of - false -> ok; - true -> - {ok, [{application, '$(1)', L0}]} = file:consult(App), - Mods = filelib:fold_files("$(call core_native_path,$(DEPS_DIR)/$1/src)", "\\\\.erl$$", true, - fun (F, Acc) -> [list_to_atom(filename:rootname(filename:basename(F)))|Acc] end, []), - L = lists:keystore(modules, 1, L0, {modules, Mods}), - ok = file:write_file(App, io_lib:format("~p.~n", [{application, '$(1)', L}])) - end - end, - UpdateModules("$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"), - halt() -endef - -define dep_autopatch_appsrc_script.erl - AppSrc = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", - AppSrcScript = AppSrc ++ ".script", - Bindings = erl_eval:new_bindings(), - {ok, Conf} = file:script(AppSrcScript, Bindings), - ok = file:write_file(AppSrc, io_lib:format("~p.~n", [Conf])), - halt() -endef - -define dep_autopatch_appsrc.erl - AppSrcOut = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", - AppSrcIn = case filelib:is_regular(AppSrcOut) of false -> "$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"; true -> AppSrcOut end, - case filelib:is_regular(AppSrcIn) of - false -> ok; - true -> - {ok, [{application, $(1), L0}]} = file:consult(AppSrcIn), - L1 = lists:keystore(modules, 1, L0, {modules, []}), - L2 = case lists:keyfind(vsn, 1, L1) of {_, git} -> lists:keyreplace(vsn, 1, L1, {vsn, "git"}); _ -> L1 end, - L3 = case lists:keyfind(registered, 1, L2) of false -> [{registered, []}|L2]; _ -> L2 end, - ok = file:write_file(AppSrcOut, io_lib:format("~p.~n", [{application, $(1), L3}])), - case AppSrcOut of AppSrcIn -> ok; _ -> ok = file:delete(AppSrcIn) end - end, - halt() -endef - -define dep_fetch_git - git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ - cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1)); -endef - -define dep_fetch_git-submodule - git submodule update --init -- $(DEPS_DIR)/$1; -endef - -define dep_fetch_hg - hg clone -q -U $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ - cd $(DEPS_DIR)/$(call dep_name,$(1)) && hg update -q $(call dep_commit,$(1)); -endef - -define dep_fetch_svn - svn checkout -q $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); -endef - -define dep_fetch_cp - cp -R $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); -endef - -define dep_fetch_hex.erl - ssl:start(), - inets:start(), - {ok, {{_, 200, _}, _, Body}} = httpc:request(get, - {"https://s3.amazonaws.com/s3.hex.pm/tarballs/$(1)-$(2).tar", []}, - [], [{body_format, binary}]), - {ok, Files} = erl_tar:extract({binary, Body}, [memory]), - {_, Source} = lists:keyfind("contents.tar.gz", 1, Files), - ok = erl_tar:extract({binary, Source}, [{cwd, "$(call core_native_path,$(DEPS_DIR)/$1)"}, compressed]), - halt() -endef - -# Hex only has a package version. No need to look in the Erlang.mk packages. -define dep_fetch_hex - $(call erlang,$(call dep_fetch_hex.erl,$(1),$(strip $(word 2,$(dep_$(1)))))); -endef - -define dep_fetch_fail - echo "Error: Unknown or invalid dependency: $(1)." >&2; \ - exit 78; -endef - -# Kept for compatibility purposes with older Erlang.mk configuration. -define dep_fetch_legacy - $(warning WARNING: '$(1)' dependency configuration uses deprecated format.) \ - git clone -q -n -- $(word 1,$(dep_$(1))) $(DEPS_DIR)/$(1); \ - cd $(DEPS_DIR)/$(1) && git checkout -q $(if $(word 2,$(dep_$(1))),$(word 2,$(dep_$(1))),master); -endef - -define dep_fetch - $(if $(dep_$(1)), \ - $(if $(dep_fetch_$(word 1,$(dep_$(1)))), \ - $(word 1,$(dep_$(1))), \ - $(if $(IS_DEP),legacy,fail)), \ - $(if $(filter $(1),$(PACKAGES)), \ - $(pkg_$(1)_fetch), \ - fail)) -endef - -define dep_target -$(DEPS_DIR)/$(call dep_name,$1): - $(eval DEP_NAME := $(call dep_name,$1)) - $(eval DEP_STR := $(if $(filter-out $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) - $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ - echo "Error: Dependency" $(DEP_STR) "conflicts with application found in $(APPS_DIR)/$(DEP_NAME)."; \ - exit 17; \ - fi - $(verbose) mkdir -p $(DEPS_DIR) - $(dep_verbose) $(call dep_fetch_$(strip $(call dep_fetch,$(1))),$(1)) - $(verbose) if [ -f $(DEPS_DIR)/$(1)/configure.ac -o -f $(DEPS_DIR)/$(1)/configure.in ] \ - && [ ! -f $(DEPS_DIR)/$(1)/configure ]; then \ - echo " AUTO " $(1); \ - cd $(DEPS_DIR)/$(1) && autoreconf -Wall -vif -I m4; \ - fi - - $(verbose) if [ -f $(DEPS_DIR)/$(DEP_NAME)/configure ]; then \ - echo " CONF " $(DEP_STR); \ - cd $(DEPS_DIR)/$(DEP_NAME) && ./configure; \ - fi -ifeq ($(filter $(1),$(NO_AUTOPATCH)),) - $(verbose) if [ "$(1)" = "amqp_client" -a "$(RABBITMQ_CLIENT_PATCH)" ]; then \ - if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ - echo " PATCH Downloading rabbitmq-codegen"; \ - git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ - fi; \ - if [ ! -d $(DEPS_DIR)/rabbitmq-server ]; then \ - echo " PATCH Downloading rabbitmq-server"; \ - git clone https://github.com/rabbitmq/rabbitmq-server.git $(DEPS_DIR)/rabbitmq-server; \ - fi; \ - ln -s $(DEPS_DIR)/amqp_client/deps/rabbit_common-0.0.0 $(DEPS_DIR)/rabbit_common; \ - elif [ "$(1)" = "rabbit" -a "$(RABBITMQ_SERVER_PATCH)" ]; then \ - if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ - echo " PATCH Downloading rabbitmq-codegen"; \ - git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ - fi \ - else \ - $$(call dep_autopatch,$(DEP_NAME)) \ - fi -endif -endef - -$(foreach dep,$(BUILD_DEPS) $(DEPS),$(eval $(call dep_target,$(dep)))) - -ifndef IS_APP -clean:: clean-apps - -clean-apps: - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep clean IS_APP=1 || exit $$?; \ - done - -distclean:: distclean-apps - -distclean-apps: - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep distclean IS_APP=1 || exit $$?; \ - done -endif - -ifndef SKIP_DEPS -distclean:: distclean-deps - -distclean-deps: - $(gen_verbose) rm -rf $(DEPS_DIR) -endif - -# External plugins. - -DEP_PLUGINS ?= - -define core_dep_plugin --include $(DEPS_DIR)/$(1) - -$(DEPS_DIR)/$(1): $(DEPS_DIR)/$(2) ; -endef - -$(foreach p,$(DEP_PLUGINS),\ - $(eval $(if $(findstring /,$p),\ - $(call core_dep_plugin,$p,$(firstword $(subst /, ,$p))),\ - $(call core_dep_plugin,$p/plugins.mk,$p)))) - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -# Configuration. - -DTL_FULL_PATH ?= -DTL_PATH ?= templates/ -DTL_SUFFIX ?= _dtl -DTL_OPTS ?= - -# Verbosity. - -dtl_verbose_0 = @echo " DTL " $(filter %.dtl,$(?F)); -dtl_verbose = $(dtl_verbose_$(V)) - -# Core targets. - -DTL_FILES = $(sort $(call core_find,$(DTL_PATH),*.dtl)) - -ifneq ($(DTL_FILES),) - -ifdef DTL_FULL_PATH -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(subst /,_,$(DTL_FILES:$(DTL_PATH)%=%)))) -else -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(notdir $(DTL_FILES)))) -endif - -# Rebuild templates when the Makefile changes. -$(DTL_FILES): $(MAKEFILE_LIST) - @touch $@ - -define erlydtl_compile.erl - [begin - Module0 = case "$(strip $(DTL_FULL_PATH))" of - "" -> - filename:basename(F, ".dtl"); - _ -> - "$(DTL_PATH)" ++ F2 = filename:rootname(F, ".dtl"), - re:replace(F2, "/", "_", [{return, list}, global]) - end, - Module = list_to_atom(string:to_lower(Module0) ++ "$(DTL_SUFFIX)"), - case erlydtl:compile(F, Module, [$(DTL_OPTS)] ++ [{out_dir, "ebin/"}, return_errors, {doc_root, "templates"}]) of - ok -> ok; - {ok, _} -> ok - end - end || F <- string:tokens("$(1)", " ")], - halt(). -endef - -ebin/$(PROJECT).app:: $(DTL_FILES) | ebin/ - $(if $(strip $?),\ - $(dtl_verbose) $(call erlang,$(call erlydtl_compile.erl,$?),-pa ebin/ $(DEPS_DIR)/erlydtl/ebin/)) - -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -# Verbosity. - -proto_verbose_0 = @echo " PROTO " $(filter %.proto,$(?F)); -proto_verbose = $(proto_verbose_$(V)) - -# Core targets. - -define compile_proto - $(verbose) mkdir -p ebin/ include/ - $(proto_verbose) $(call erlang,$(call compile_proto.erl,$(1))) - $(proto_verbose) erlc +debug_info -o ebin/ ebin/*.erl - $(verbose) rm ebin/*.erl -endef - -define compile_proto.erl - [begin - Dir = filename:dirname(filename:dirname(F)), - protobuffs_compile:generate_source(F, - [{output_include_dir, Dir ++ "/include"}, - {output_src_dir, Dir ++ "/ebin"}]) - end || F <- string:tokens("$(1)", " ")], - halt(). -endef - -ifneq ($(wildcard src/),) -ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.proto)) - $(if $(strip $?),$(call compile_proto,$?)) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: clean-app - -# Configuration. - -ERLC_OPTS ?= -Werror +debug_info +warn_export_vars +warn_shadow_vars \ - +warn_obsolete_guard # +bin_opt_info +warn_export_all +warn_missing_spec -COMPILE_FIRST ?= -COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .erl,$(COMPILE_FIRST))) -ERLC_EXCLUDE ?= -ERLC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .erl,$(ERLC_EXCLUDE))) - -ERLC_MIB_OPTS ?= -COMPILE_MIB_FIRST ?= -COMPILE_MIB_FIRST_PATHS = $(addprefix mibs/,$(addsuffix .mib,$(COMPILE_MIB_FIRST))) - -# Verbosity. - -app_verbose_0 = @echo " APP " $(PROJECT); -app_verbose_2 = set -x; -app_verbose = $(app_verbose_$(V)) - -appsrc_verbose_0 = @echo " APP " $(PROJECT).app.src; -appsrc_verbose_2 = set -x; -appsrc_verbose = $(appsrc_verbose_$(V)) - -makedep_verbose_0 = @echo " DEPEND" $(PROJECT).d; -makedep_verbose_2 = set -x; -makedep_verbose = $(makedep_verbose_$(V)) - -erlc_verbose_0 = @echo " ERLC " $(filter-out $(patsubst %,%.erl,$(ERLC_EXCLUDE)),\ - $(filter %.erl %.core,$(?F))); -erlc_verbose_2 = set -x; -erlc_verbose = $(erlc_verbose_$(V)) - -xyrl_verbose_0 = @echo " XYRL " $(filter %.xrl %.yrl,$(?F)); -xyrl_verbose_2 = set -x; -xyrl_verbose = $(xyrl_verbose_$(V)) - -asn1_verbose_0 = @echo " ASN1 " $(filter %.asn1,$(?F)); -asn1_verbose_2 = set -x; -asn1_verbose = $(asn1_verbose_$(V)) - -mib_verbose_0 = @echo " MIB " $(filter %.bin %.mib,$(?F)); -mib_verbose_2 = set -x; -mib_verbose = $(mib_verbose_$(V)) - -ifneq ($(wildcard src/),) - -# Targets. - -ifeq ($(wildcard ebin/test),) -app:: deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build -else -app:: clean deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build -endif - -ifeq ($(wildcard src/$(PROJECT_MOD).erl),) -define app_file -{application, $(PROJECT), [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$(1)"}$(comma)) - {modules, [$(call comma_list,$(2))]}, - {registered, []}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]} -]}. -endef -else -define app_file -{application, $(PROJECT), [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$(1)"}$(comma)) - {modules, [$(call comma_list,$(2))]}, - {registered, [$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED))]}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]}, - {mod, {$(PROJECT_MOD), []}} -]}. -endef -endif - -app-build: ebin/$(PROJECT).app - $(verbose) : - -# Source files. - -ERL_FILES = $(sort $(call core_find,src/,*.erl)) -CORE_FILES = $(sort $(call core_find,src/,*.core)) - -# ASN.1 files. - -ifneq ($(wildcard asn1/),) -ASN1_FILES = $(sort $(call core_find,asn1/,*.asn1)) -ERL_FILES += $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) - -define compile_asn1 - $(verbose) mkdir -p include/ - $(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(1) - $(verbose) mv asn1/*.erl src/ - $(verbose) mv asn1/*.hrl include/ - $(verbose) mv asn1/*.asn1db include/ -endef - -$(PROJECT).d:: $(ASN1_FILES) - $(if $(strip $?),$(call compile_asn1,$?)) -endif - -# SNMP MIB files. - -ifneq ($(wildcard mibs/),) -MIB_FILES = $(sort $(call core_find,mibs/,*.mib)) - -$(PROJECT).d:: $(COMPILE_MIB_FIRST_PATHS) $(MIB_FILES) - $(verbose) mkdir -p include/ priv/mibs/ - $(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ -I priv/mibs/ $? - $(mib_verbose) erlc -o include/ -- $(addprefix priv/mibs/,$(patsubst %.mib,%.bin,$(notdir $?))) -endif - -# Leex and Yecc files. - -XRL_FILES = $(sort $(call core_find,src/,*.xrl)) -XRL_ERL_FILES = $(addprefix src/,$(patsubst %.xrl,%.erl,$(notdir $(XRL_FILES)))) -ERL_FILES += $(XRL_ERL_FILES) - -YRL_FILES = $(sort $(call core_find,src/,*.yrl)) -YRL_ERL_FILES = $(addprefix src/,$(patsubst %.yrl,%.erl,$(notdir $(YRL_FILES)))) -ERL_FILES += $(YRL_ERL_FILES) - -$(PROJECT).d:: $(XRL_FILES) $(YRL_FILES) - $(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $?) - -# Erlang and Core Erlang files. - -define makedep.erl - E = ets:new(makedep, [bag]), - G = digraph:new([acyclic]), - ErlFiles = lists:usort(string:tokens("$(ERL_FILES)", " ")), - Modules = [{list_to_atom(filename:basename(F, ".erl")), F} || F <- ErlFiles], - Add = fun (Mod, Dep) -> - case lists:keyfind(Dep, 1, Modules) of - false -> ok; - {_, DepFile} -> - {_, ModFile} = lists:keyfind(Mod, 1, Modules), - ets:insert(E, {ModFile, DepFile}), - digraph:add_vertex(G, Mod), - digraph:add_vertex(G, Dep), - digraph:add_edge(G, Mod, Dep) - end - end, - AddHd = fun (F, Mod, DepFile) -> - case file:open(DepFile, [read]) of - {error, enoent} -> ok; - {ok, Fd} -> - F(F, Fd, Mod), - {_, ModFile} = lists:keyfind(Mod, 1, Modules), - ets:insert(E, {ModFile, DepFile}) - end - end, - Attr = fun - (F, Mod, behavior, Dep) -> Add(Mod, Dep); - (F, Mod, behaviour, Dep) -> Add(Mod, Dep); - (F, Mod, compile, {parse_transform, Dep}) -> Add(Mod, Dep); - (F, Mod, compile, Opts) when is_list(Opts) -> - case proplists:get_value(parse_transform, Opts) of - undefined -> ok; - Dep -> Add(Mod, Dep) - end; - (F, Mod, include, Hrl) -> - case filelib:is_file("include/" ++ Hrl) of - true -> AddHd(F, Mod, "include/" ++ Hrl); - false -> - case filelib:is_file("src/" ++ Hrl) of - true -> AddHd(F, Mod, "src/" ++ Hrl); - false -> false - end - end; - (F, Mod, include_lib, "$1/include/" ++ Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); - (F, Mod, include_lib, Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); - (F, Mod, import, {Imp, _}) -> - case filelib:is_file("src/" ++ atom_to_list(Imp) ++ ".erl") of - false -> ok; - true -> Add(Mod, Imp) - end; - (_, _, _, _) -> ok - end, - MakeDepend = fun(F, Fd, Mod) -> - case io:parse_erl_form(Fd, undefined) of - {ok, {attribute, _, Key, Value}, _} -> - Attr(F, Mod, Key, Value), - F(F, Fd, Mod); - {eof, _} -> - file:close(Fd); - _ -> - F(F, Fd, Mod) - end - end, - [begin - Mod = list_to_atom(filename:basename(F, ".erl")), - {ok, Fd} = file:open(F, [read]), - MakeDepend(MakeDepend, Fd, Mod) - end || F <- ErlFiles], - Depend = sofs:to_external(sofs:relation_to_family(sofs:relation(ets:tab2list(E)))), - CompileFirst = [X || X <- lists:reverse(digraph_utils:topsort(G)), [] =/= digraph:in_neighbours(G, X)], - ok = file:write_file("$(1)", [ - [[F, "::", [[" ", D] || D <- Deps], "; @touch \$$@\n"] || {F, Deps} <- Depend], - "\nCOMPILE_FIRST +=", [[" ", atom_to_list(CF)] || CF <- CompileFirst], "\n" - ]), - halt() -endef - -ifeq ($(if $(NO_MAKEDEP),$(wildcard $(PROJECT).d),),) -$(PROJECT).d:: $(ERL_FILES) $(call core_find,include/,*.hrl) - $(makedep_verbose) $(call erlang,$(call makedep.erl,$@)) -endif - -# Rebuild everything when the Makefile changes. -$(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(MAKEFILE_LIST) - @touch $@ - --include $(PROJECT).d - -ebin/$(PROJECT).app:: ebin/ - -ebin/: - $(verbose) mkdir -p ebin/ - -define compile_erl - $(erlc_verbose) erlc -v $(if $(IS_DEP),$(filter-out -Werror,$(ERLC_OPTS)),$(ERLC_OPTS)) -o ebin/ \ - -pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),$(COMPILE_FIRST_PATHS) $(1)) -endef - -ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) - $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) - $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) - $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null || true)) - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ - $(filter-out $(ERLC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) -ifeq ($(wildcard src/$(PROJECT).app.src),) - $(app_verbose) printf "$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES))))" \ - > ebin/$(PROJECT).app -else - $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ - echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk README for instructions." >&2; \ - exit 1; \ - fi - $(appsrc_verbose) cat src/$(PROJECT).app.src \ - | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ - | sed "s/{vsn,[[:space:]]*\"git\"}/{vsn, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ - > ebin/$(PROJECT).app -endif - -clean:: clean-app - -clean-app: - $(gen_verbose) rm -rf $(PROJECT).d ebin/ priv/mibs/ $(XRL_ERL_FILES) $(YRL_ERL_FILES) \ - $(addprefix include/,$(patsubst %.mib,%.hrl,$(notdir $(MIB_FILES)))) \ - $(addprefix include/,$(patsubst %.asn1,%.hrl,$(notdir $(ASN1_FILES)))) \ - $(addprefix include/,$(patsubst %.asn1,%.asn1db,$(notdir $(ASN1_FILES)))) \ - $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) - -endif - -# Copyright (c) 2015, Viktor Söderqvist -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: docs-deps - -# Configuration. - -ALL_DOC_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(DOC_DEPS)) - -# Targets. - -$(foreach dep,$(DOC_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -doc-deps: -else -doc-deps: $(ALL_DOC_DEPS_DIRS) - $(verbose) for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep; done -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: rel-deps - -# Configuration. - -ALL_REL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(REL_DEPS)) - -# Targets. - -$(foreach dep,$(REL_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -rel-deps: -else -rel-deps: $(ALL_REL_DEPS_DIRS) - $(verbose) for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: test-deps test-dir test-build clean-test-dir - -# Configuration. - -TEST_DIR ?= $(CURDIR)/test - -ALL_TEST_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(TEST_DEPS)) - -TEST_ERLC_OPTS ?= +debug_info +warn_export_vars +warn_shadow_vars +warn_obsolete_guard -TEST_ERLC_OPTS += -DTEST=1 - -# Targets. - -$(foreach dep,$(TEST_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -test-deps: -else -test-deps: $(ALL_TEST_DEPS_DIRS) - $(verbose) for dep in $(ALL_TEST_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1; done -endif - -ifneq ($(wildcard $(TEST_DIR)),) -test-dir: - $(gen_verbose) erlc -v $(TEST_ERLC_OPTS) -I include/ -o $(TEST_DIR) \ - $(call core_find,$(TEST_DIR)/,*.erl) -pa ebin/ -endif - -ifeq ($(wildcard src),) -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: clean deps test-deps - $(verbose) $(MAKE) --no-print-directory test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" -else -ifeq ($(wildcard ebin/test),) -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: clean deps test-deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" - $(gen_verbose) touch ebin/test -else -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: deps test-deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" -endif - -clean:: clean-test-dir - -clean-test-dir: -ifneq ($(wildcard $(TEST_DIR)/*.beam),) - $(gen_verbose) rm -f $(TEST_DIR)/*.beam -endif -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: rebar.config - -# We strip out -Werror because we don't want to fail due to -# warnings when used as a dependency. - -compat_prepare_erlc_opts = $(shell echo "$1" | sed 's/, */,/g') - -define compat_convert_erlc_opts -$(if $(filter-out -Werror,$1),\ - $(if $(findstring +,$1),\ - $(shell echo $1 | cut -b 2-))) -endef - -define compat_erlc_opts_to_list -[$(call comma_list,$(foreach o,$(call compat_prepare_erlc_opts,$1),$(call compat_convert_erlc_opts,$o)))] -endef - -define compat_rebar_config -{deps, [ -$(call comma_list,$(foreach d,$(DEPS),\ - $(if $(filter hex,$(call dep_fetch,$d)),\ - {$(call dep_name,$d)$(comma)"$(call dep_repo,$d)"},\ - {$(call dep_name,$d)$(comma)".*"$(comma){git,"$(call dep_repo,$d)"$(comma)"$(call dep_commit,$d)"}}))) -]}. -{erl_opts, $(call compat_erlc_opts_to_list,$(ERLC_OPTS))}. -endef - -$(eval _compat_rebar_config = $$(compat_rebar_config)) -$(eval export _compat_rebar_config) - -rebar.config: - $(gen_verbose) echo "$${_compat_rebar_config}" > rebar.config - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: asciidoc asciidoc-guide asciidoc-manual install-asciidoc distclean-asciidoc - -MAN_INSTALL_PATH ?= /usr/local/share/man -MAN_SECTIONS ?= 3 7 - -docs:: asciidoc - -asciidoc: asciidoc-guide asciidoc-manual - -ifeq ($(wildcard doc/src/guide/book.asciidoc),) -asciidoc-guide: -else -asciidoc-guide: distclean-asciidoc doc-deps - a2x -v -f pdf doc/src/guide/book.asciidoc && mv doc/src/guide/book.pdf doc/guide.pdf - a2x -v -f chunked doc/src/guide/book.asciidoc && mv doc/src/guide/book.chunked/ doc/html/ -endif - -ifeq ($(wildcard doc/src/manual/*.asciidoc),) -asciidoc-manual: -else -asciidoc-manual: distclean-asciidoc doc-deps - for f in doc/src/manual/*.asciidoc ; do \ - a2x -v -f manpage $$f ; \ - done - for s in $(MAN_SECTIONS); do \ - mkdir -p doc/man$$s/ ; \ - mv doc/src/manual/*.$$s doc/man$$s/ ; \ - gzip doc/man$$s/*.$$s ; \ - done - -install-docs:: install-asciidoc - -install-asciidoc: asciidoc-manual - for s in $(MAN_SECTIONS); do \ - mkdir -p $(MAN_INSTALL_PATH)/man$$s/ ; \ - install -g `id -u` -o `id -g` -m 0644 doc/man$$s/*.gz $(MAN_INSTALL_PATH)/man$$s/ ; \ - done -endif - -distclean:: distclean-asciidoc - -distclean-asciidoc: - $(gen_verbose) rm -rf doc/html/ doc/guide.pdf doc/man3/ doc/man7/ - -# Copyright (c) 2014-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: bootstrap bootstrap-lib bootstrap-rel new list-templates - -# Core targets. - -help:: - $(verbose) printf "%s\n" "" \ - "Bootstrap targets:" \ - " bootstrap Generate a skeleton of an OTP application" \ - " bootstrap-lib Generate a skeleton of an OTP library" \ - " bootstrap-rel Generate the files needed to build a release" \ - " new-app in=NAME Create a new local OTP application NAME" \ - " new-lib in=NAME Create a new local OTP library NAME" \ - " new t=TPL n=NAME Generate a module NAME based on the template TPL" \ - " new t=T n=N in=APP Generate a module NAME based on the template TPL in APP" \ - " list-templates List available templates" - -# Bootstrap templates. - -define bs_appsrc -{application, $p, [ - {description, ""}, - {vsn, "0.1.0"}, - {id, "git"}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]}, - {mod, {$p_app, []}}, - {env, []} -]}. -endef - -define bs_appsrc_lib -{application, $p, [ - {description, ""}, - {vsn, "0.1.0"}, - {id, "git"}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]} -]}. -endef - -# To prevent autocompletion issues with ZSH, we add "include erlang.mk" -# separately during the actual bootstrap. -ifdef SP -define bs_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -# Whitespace to be used when creating files from templates. -SP = $(SP) - -endef -else -define bs_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -endef -endif - -define bs_apps_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(APPS_DIR)/app)/erlang.mk -endef - -define bs_app --module($p_app). --behaviour(application). - --export([start/2]). --export([stop/1]). - -start(_Type, _Args) -> - $p_sup:start_link(). - -stop(_State) -> - ok. -endef - -define bs_relx_config -{release, {$p_release, "1"}, [$p]}. -{extended_start_script, true}. -{sys_config, "rel/sys.config"}. -{vm_args, "rel/vm.args"}. -endef - -define bs_sys_config -[ -]. -endef - -define bs_vm_args --name $p@127.0.0.1 --setcookie $p --heart -endef - -# Normal templates. - -define tpl_supervisor --module($(n)). --behaviour(supervisor). - --export([start_link/0]). --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - Procs = [], - {ok, {{one_for_one, 1, 5}, Procs}}. -endef - -define tpl_gen_server --module($(n)). --behaviour(gen_server). - -%% API. --export([start_link/0]). - -%% gen_server. --export([init/1]). --export([handle_call/3]). --export([handle_cast/2]). --export([handle_info/2]). --export([terminate/2]). --export([code_change/3]). - --record(state, { -}). - -%% API. - --spec start_link() -> {ok, pid()}. -start_link() -> - gen_server:start_link(?MODULE, [], []). - -%% gen_server. - -init([]) -> - {ok, #state{}}. - -handle_call(_Request, _From, State) -> - {reply, ignored, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. -endef - -define tpl_module --module($(n)). --export([]). -endef - -define tpl_cowboy_http --module($(n)). --behaviour(cowboy_http_handler). - --export([init/3]). --export([handle/2]). --export([terminate/3]). - --record(state, { -}). - -init(_, Req, _Opts) -> - {ok, Req, #state{}}. - -handle(Req, State=#state{}) -> - {ok, Req2} = cowboy_req:reply(200, Req), - {ok, Req2, State}. - -terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_gen_fsm --module($(n)). --behaviour(gen_fsm). - -%% API. --export([start_link/0]). - -%% gen_fsm. --export([init/1]). --export([state_name/2]). --export([handle_event/3]). --export([state_name/3]). --export([handle_sync_event/4]). --export([handle_info/3]). --export([terminate/3]). --export([code_change/4]). - --record(state, { -}). - -%% API. - --spec start_link() -> {ok, pid()}. -start_link() -> - gen_fsm:start_link(?MODULE, [], []). - -%% gen_fsm. - -init([]) -> - {ok, state_name, #state{}}. - -state_name(_Event, StateData) -> - {next_state, state_name, StateData}. - -handle_event(_Event, StateName, StateData) -> - {next_state, StateName, StateData}. - -state_name(_Event, _From, StateData) -> - {reply, ignored, state_name, StateData}. - -handle_sync_event(_Event, _From, StateName, StateData) -> - {reply, ignored, StateName, StateData}. - -handle_info(_Info, StateName, StateData) -> - {next_state, StateName, StateData}. - -terminate(_Reason, _StateName, _StateData) -> - ok. - -code_change(_OldVsn, StateName, StateData, _Extra) -> - {ok, StateName, StateData}. -endef - -define tpl_cowboy_loop --module($(n)). --behaviour(cowboy_loop_handler). - --export([init/3]). --export([info/3]). --export([terminate/3]). - --record(state, { -}). - -init(_, Req, _Opts) -> - {loop, Req, #state{}, 5000, hibernate}. - -info(_Info, Req, State) -> - {loop, Req, State, hibernate}. - -terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_cowboy_rest --module($(n)). - --export([init/3]). --export([content_types_provided/2]). --export([get_html/2]). - -init(_, _Req, _Opts) -> - {upgrade, protocol, cowboy_rest}. - -content_types_provided(Req, State) -> - {[{{<<"text">>, <<"html">>, '*'}, get_html}], Req, State}. - -get_html(Req, State) -> - {<<"This is REST!">>, Req, State}. -endef - -define tpl_cowboy_ws --module($(n)). --behaviour(cowboy_websocket_handler). - --export([init/3]). --export([websocket_init/3]). --export([websocket_handle/3]). --export([websocket_info/3]). --export([websocket_terminate/3]). - --record(state, { -}). - -init(_, _, _) -> - {upgrade, protocol, cowboy_websocket}. - -websocket_init(_, Req, _Opts) -> - Req2 = cowboy_req:compact(Req), - {ok, Req2, #state{}}. - -websocket_handle({text, Data}, Req, State) -> - {reply, {text, Data}, Req, State}; -websocket_handle({binary, Data}, Req, State) -> - {reply, {binary, Data}, Req, State}; -websocket_handle(_Frame, Req, State) -> - {ok, Req, State}. - -websocket_info(_Info, Req, State) -> - {ok, Req, State}. - -websocket_terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_ranch_protocol --module($(n)). --behaviour(ranch_protocol). - --export([start_link/4]). --export([init/4]). - --type opts() :: []. --export_type([opts/0]). - --record(state, { - socket :: inet:socket(), - transport :: module() -}). - -start_link(Ref, Socket, Transport, Opts) -> - Pid = spawn_link(?MODULE, init, [Ref, Socket, Transport, Opts]), - {ok, Pid}. - --spec init(ranch:ref(), inet:socket(), module(), opts()) -> ok. -init(Ref, Socket, Transport, _Opts) -> - ok = ranch:accept_ack(Ref), - loop(#state{socket=Socket, transport=Transport}). - -loop(State) -> - loop(State). -endef - -# Plugin-specific targets. - -define render_template - $(verbose) printf -- '$(subst $(newline),\n,$(subst %,%%,$(subst ','\'',$(subst $(tab),$(WS),$(call $(1))))))\n' > $(2) -endef - -ifndef WS -ifdef SP -WS = $(subst a,,a $(wordlist 1,$(SP),a a a a a a a a a a a a a a a a a a a a)) -else -WS = $(tab) -endif -endif - -bootstrap: -ifneq ($(wildcard src/),) - $(error Error: src/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(eval n := $(PROJECT)_sup) - $(call render_template,bs_Makefile,Makefile) - $(verbose) echo "include erlang.mk" >> Makefile - $(verbose) mkdir src/ -ifdef LEGACY - $(call render_template,bs_appsrc,src/$(PROJECT).app.src) -endif - $(call render_template,bs_app,src/$(PROJECT)_app.erl) - $(call render_template,tpl_supervisor,src/$(PROJECT)_sup.erl) - -bootstrap-lib: -ifneq ($(wildcard src/),) - $(error Error: src/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(call render_template,bs_Makefile,Makefile) - $(verbose) echo "include erlang.mk" >> Makefile - $(verbose) mkdir src/ -ifdef LEGACY - $(call render_template,bs_appsrc_lib,src/$(PROJECT).app.src) -endif - -bootstrap-rel: -ifneq ($(wildcard relx.config),) - $(error Error: relx.config already exists) -endif -ifneq ($(wildcard rel/),) - $(error Error: rel/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(call render_template,bs_relx_config,relx.config) - $(verbose) mkdir rel/ - $(call render_template,bs_sys_config,rel/sys.config) - $(call render_template,bs_vm_args,rel/vm.args) - -new-app: -ifndef in - $(error Usage: $(MAKE) new-app in=APP) -endif -ifneq ($(wildcard $(APPS_DIR)/$in),) - $(error Error: Application $in already exists) -endif - $(eval p := $(in)) - $(eval n := $(in)_sup) - $(verbose) mkdir -p $(APPS_DIR)/$p/src/ - $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) -ifdef LEGACY - $(call render_template,bs_appsrc,$(APPS_DIR)/$p/src/$p.app.src) -endif - $(call render_template,bs_app,$(APPS_DIR)/$p/src/$p_app.erl) - $(call render_template,tpl_supervisor,$(APPS_DIR)/$p/src/$p_sup.erl) - -new-lib: -ifndef in - $(error Usage: $(MAKE) new-lib in=APP) -endif -ifneq ($(wildcard $(APPS_DIR)/$in),) - $(error Error: Application $in already exists) -endif - $(eval p := $(in)) - $(verbose) mkdir -p $(APPS_DIR)/$p/src/ - $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) -ifdef LEGACY - $(call render_template,bs_appsrc_lib,$(APPS_DIR)/$p/src/$p.app.src) -endif - -new: -ifeq ($(wildcard src/)$(in),) - $(error Error: src/ directory does not exist) -endif -ifndef t - $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) -endif -ifndef tpl_$(t) - $(error Unknown template) -endif -ifndef n - $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) -endif -ifdef in - $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new t=$t n=$n in= -else - $(call render_template,tpl_$(t),src/$(n).erl) -endif - -list-templates: - $(verbose) echo Available templates: $(sort $(patsubst tpl_%,%,$(filter tpl_%,$(.VARIABLES)))) - -# Copyright (c) 2014-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: clean-c_src distclean-c_src-env - -# Configuration. - -C_SRC_DIR ?= $(CURDIR)/c_src -C_SRC_ENV ?= $(C_SRC_DIR)/env.mk -C_SRC_OUTPUT ?= $(CURDIR)/priv/$(PROJECT) -C_SRC_TYPE ?= shared - -# System type and C compiler/flags. - -ifeq ($(PLATFORM),msys2) - C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= .exe - C_SRC_OUTPUT_SHARED_EXTENSION ?= .dll -else - C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= - C_SRC_OUTPUT_SHARED_EXTENSION ?= .so -endif - -ifeq ($(C_SRC_TYPE),shared) - C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_SHARED_EXTENSION) -else - C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_EXECUTABLE_EXTENSION) -endif - -ifeq ($(PLATFORM),msys2) -# We hardcode the compiler used on MSYS2. The default CC=cc does -# not produce working code. The "gcc" MSYS2 package also doesn't. - CC = /mingw64/bin/gcc - export CC - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -else ifeq ($(PLATFORM),darwin) - CC ?= cc - CFLAGS ?= -O3 -std=c99 -arch x86_64 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -arch x86_64 -finline-functions -Wall - LDFLAGS ?= -arch x86_64 -flat_namespace -undefined suppress -else ifeq ($(PLATFORM),freebsd) - CC ?= cc - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -else ifeq ($(PLATFORM),linux) - CC ?= gcc - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -endif - -ifneq ($(PLATFORM),msys2) - CFLAGS += -fPIC - CXXFLAGS += -fPIC -endif - -CFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" -CXXFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" - -LDLIBS += -L"$(ERL_INTERFACE_LIB_DIR)" -lerl_interface -lei - -# Verbosity. - -c_verbose_0 = @echo " C " $(?F); -c_verbose = $(c_verbose_$(V)) - -cpp_verbose_0 = @echo " CPP " $(?F); -cpp_verbose = $(cpp_verbose_$(V)) - -link_verbose_0 = @echo " LD " $(@F); -link_verbose = $(link_verbose_$(V)) - -# Targets. - -ifeq ($(wildcard $(C_SRC_DIR)),) -else ifneq ($(wildcard $(C_SRC_DIR)/Makefile),) -app:: app-c_src - -test-build:: app-c_src - -app-c_src: - $(MAKE) -C $(C_SRC_DIR) - -clean:: - $(MAKE) -C $(C_SRC_DIR) clean - -else - -ifeq ($(SOURCES),) -SOURCES := $(sort $(foreach pat,*.c *.C *.cc *.cpp,$(call core_find,$(C_SRC_DIR)/,$(pat)))) -endif -OBJECTS = $(addsuffix .o, $(basename $(SOURCES))) - -COMPILE_C = $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c -COMPILE_CPP = $(cpp_verbose) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c - -app:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) - -test-build:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) - -$(C_SRC_OUTPUT_FILE): $(OBJECTS) - $(verbose) mkdir -p priv/ - $(link_verbose) $(CC) $(OBJECTS) \ - $(LDFLAGS) $(if $(filter $(C_SRC_TYPE),shared),-shared) $(LDLIBS) \ - -o $(C_SRC_OUTPUT_FILE) - -%.o: %.c - $(COMPILE_C) $(OUTPUT_OPTION) $< - -%.o: %.cc - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -%.o: %.C - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -%.o: %.cpp - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -clean:: clean-c_src - -clean-c_src: - $(gen_verbose) rm -f $(C_SRC_OUTPUT_FILE) $(OBJECTS) - -endif - -ifneq ($(wildcard $(C_SRC_DIR)),) -$(C_SRC_ENV): - $(verbose) $(ERL) -eval "file:write_file(\"$(call core_native_path,$(C_SRC_ENV))\", \ - io_lib:format( \ - \"ERTS_INCLUDE_DIR ?= ~s/erts-~s/include/~n\" \ - \"ERL_INTERFACE_INCLUDE_DIR ?= ~s~n\" \ - \"ERL_INTERFACE_LIB_DIR ?= ~s~n\", \ - [code:root_dir(), erlang:system_info(version), \ - code:lib_dir(erl_interface, include), \ - code:lib_dir(erl_interface, lib)])), \ - halt()." - -distclean:: distclean-c_src-env - -distclean-c_src-env: - $(gen_verbose) rm -f $(C_SRC_ENV) - --include $(C_SRC_ENV) -endif - -# Templates. - -define bs_c_nif -#include "erl_nif.h" - -static int loads = 0; - -static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - /* Initialize private data. */ - *priv_data = NULL; - - loads++; - - return 0; -} - -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) -{ - /* Convert the private data to the new version. */ - *priv_data = *old_priv_data; - - loads++; - - return 0; -} - -static void unload(ErlNifEnv* env, void* priv_data) -{ - if (loads == 1) { - /* Destroy the private data. */ - } - - loads--; -} - -static ERL_NIF_TERM hello(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - if (enif_is_atom(env, argv[0])) { - return enif_make_tuple2(env, - enif_make_atom(env, "hello"), - argv[0]); - } - - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); -} - -static ErlNifFunc nif_funcs[] = { - {"hello", 1, hello} -}; - -ERL_NIF_INIT($n, nif_funcs, load, NULL, upgrade, unload) -endef - -define bs_erl_nif --module($n). - --export([hello/1]). - --on_load(on_load/0). -on_load() -> - PrivDir = case code:priv_dir(?MODULE) of - {error, _} -> - AppPath = filename:dirname(filename:dirname(code:which(?MODULE))), - filename:join(AppPath, "priv"); - Path -> - Path - end, - erlang:load_nif(filename:join(PrivDir, atom_to_list(?MODULE)), 0). - -hello(_) -> - erlang:nif_error({not_loaded, ?MODULE}). -endef - -new-nif: -ifneq ($(wildcard $(C_SRC_DIR)/$n.c),) - $(error Error: $(C_SRC_DIR)/$n.c already exists) -endif -ifneq ($(wildcard src/$n.erl),) - $(error Error: src/$n.erl already exists) -endif -ifdef in - $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new-nif n=$n in= -else - $(verbose) mkdir -p $(C_SRC_DIR) src/ - $(call render_template,bs_c_nif,$(C_SRC_DIR)/$n.c) - $(call render_template,bs_erl_nif,src/$n.erl) -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: ci ci-setup distclean-kerl - -KERL ?= $(CURDIR)/kerl -export KERL - -KERL_URL ?= https://raw.githubusercontent.com/yrashk/kerl/master/kerl - -OTP_GIT ?= https://github.com/erlang/otp - -CI_INSTALL_DIR ?= $(HOME)/erlang -CI_OTP ?= - -ifeq ($(strip $(CI_OTP)),) -ci:: -else -ci:: $(addprefix ci-,$(CI_OTP)) - -ci-prepare: $(addprefix $(CI_INSTALL_DIR)/,$(CI_OTP)) - -ci-setup:: - -ci_verbose_0 = @echo " CI " $(1); -ci_verbose = $(ci_verbose_$(V)) - -define ci_target -ci-$(1): $(CI_INSTALL_DIR)/$(1) - $(ci_verbose) \ - PATH="$(CI_INSTALL_DIR)/$(1)/bin:$(PATH)" \ - CI_OTP_RELEASE="$(1)" \ - CT_OPTS="-label $(1)" \ - $(MAKE) clean ci-setup tests -endef - -$(foreach otp,$(CI_OTP),$(eval $(call ci_target,$(otp)))) - -define ci_otp_target -ifeq ($(wildcard $(CI_INSTALL_DIR)/$(1)),) -$(CI_INSTALL_DIR)/$(1): $(KERL) - $(KERL) build git $(OTP_GIT) $(1) $(1) - $(KERL) install $(1) $(CI_INSTALL_DIR)/$(1) -endif -endef - -$(foreach otp,$(CI_OTP),$(eval $(call ci_otp_target,$(otp)))) - -$(KERL): - $(gen_verbose) $(call core_http_get,$(KERL),$(KERL_URL)) - $(verbose) chmod +x $(KERL) - -help:: - $(verbose) printf "%s\n" "" \ - "Continuous Integration targets:" \ - " ci Run '$(MAKE) tests' on all configured Erlang versions." \ - "" \ - "The CI_OTP variable must be defined with the Erlang versions" \ - "that must be tested. For example: CI_OTP = OTP-17.3.4 OTP-17.5.3" - -distclean:: distclean-kerl - -distclean-kerl: - $(gen_verbose) rm -rf $(KERL) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: ct apps-ct distclean-ct - -# Configuration. - -CT_OPTS ?= -ifneq ($(wildcard $(TEST_DIR)),) - CT_SUITES ?= $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*_SUITE.erl)))) -else - CT_SUITES ?= -endif - -# Core targets. - -tests:: ct - -distclean:: distclean-ct - -help:: - $(verbose) printf "%s\n" "" \ - "Common_test targets:" \ - " ct Run all the common_test suites for this project" \ - "" \ - "All your common_test suites have their associated targets." \ - "A suite named http_SUITE can be ran using the ct-http target." - -# Plugin-specific targets. - -CT_RUN = ct_run \ - -no_auto_compile \ - -noinput \ - -pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin $(DEPS_DIR)/gen_rpc/_build/dev/lib/*/ebin $(APPS_DIR)/*/ebin $(TEST_DIR) \ - -dir $(TEST_DIR) \ - -logdir $(CURDIR)/logs - -ifeq ($(CT_SUITES),) -ct: $(if $(IS_APP),,apps-ct) -else -ct: test-build $(if $(IS_APP),,apps-ct) - $(verbose) mkdir -p $(CURDIR)/logs/ - $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS) -endif - -ifneq ($(ALL_APPS_DIRS),) -define ct_app_target -apps-ct-$1: - $(MAKE) -C $1 ct IS_APP=1 -endef - -$(foreach app,$(ALL_APPS_DIRS),$(eval $(call ct_app_target,$(app)))) - -apps-ct: test-build $(addprefix apps-ct-,$(ALL_APPS_DIRS)) -endif - -ifndef t -CT_EXTRA = -else -ifeq (,$(findstring :,$t)) -CT_EXTRA = -group $t -else -t_words = $(subst :, ,$t) -CT_EXTRA = -group $(firstword $(t_words)) -case $(lastword $(t_words)) -endif -endif - -define ct_suite_target -ct-$(1): test-build - $(verbose) mkdir -p $(CURDIR)/logs/ - $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS) -endef - -$(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test)))) - -distclean-ct: - $(gen_verbose) rm -rf $(CURDIR)/logs/ - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: plt distclean-plt dialyze - -# Configuration. - -DIALYZER_PLT ?= $(CURDIR)/.$(PROJECT).plt -export DIALYZER_PLT - -PLT_APPS ?= -DIALYZER_DIRS ?= --src -r $(wildcard src) $(ALL_APPS_DIRS) -DIALYZER_OPTS ?= -Werror_handling -Wrace_conditions -Wunmatched_returns # -Wunderspecs - -# Core targets. - -check:: dialyze - -distclean:: distclean-plt - -help:: - $(verbose) printf "%s\n" "" \ - "Dialyzer targets:" \ - " plt Build a PLT file for this project" \ - " dialyze Analyze the project using Dialyzer" - -# Plugin-specific targets. - -define filter_opts.erl - Opts = binary:split(<<"$1">>, <<"-">>, [global]), - Filtered = lists:reverse(lists:foldl(fun - (O = <<"pa ", _/bits>>, Acc) -> [O|Acc]; - (O = <<"D ", _/bits>>, Acc) -> [O|Acc]; - (O = <<"I ", _/bits>>, Acc) -> [O|Acc]; - (_, Acc) -> Acc - end, [], Opts)), - io:format("~s~n", [[["-", O] || O <- Filtered]]), - halt(). -endef - -$(DIALYZER_PLT): deps app - $(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS) - -plt: $(DIALYZER_PLT) - -distclean-plt: - $(gen_verbose) rm -f $(DIALYZER_PLT) - -ifneq ($(wildcard $(DIALYZER_PLT)),) -dialyze: -else -dialyze: $(DIALYZER_PLT) -endif - $(verbose) dialyzer --no_native `$(call erlang,$(call filter_opts.erl,$(ERLC_OPTS)))` $(DIALYZER_DIRS) $(DIALYZER_OPTS) - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-edoc edoc - -# Configuration. - -EDOC_OPTS ?= - -# Core targets. - -ifneq ($(wildcard doc/overview.edoc),) -docs:: edoc -endif - -distclean:: distclean-edoc - -# Plugin-specific targets. - -edoc: distclean-edoc doc-deps - $(gen_verbose) $(ERL) -eval 'edoc:application($(PROJECT), ".", [$(EDOC_OPTS)]), halt().' - -distclean-edoc: - $(gen_verbose) rm -f doc/*.css doc/*.html doc/*.png doc/edoc-info - -# Copyright (c) 2014 Dave Cottlehuber -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-escript escript - -# Configuration. - -ESCRIPT_NAME ?= $(PROJECT) -ESCRIPT_FILE ?= $(ESCRIPT_NAME) - -ESCRIPT_COMMENT ?= This is an -*- erlang -*- file - -ESCRIPT_BEAMS ?= "ebin/*", "deps/*/ebin/*" -ESCRIPT_SYS_CONFIG ?= "rel/sys.config" -ESCRIPT_EMU_ARGS ?= -pa . \ - -sasl errlog_type error \ - -escript main $(ESCRIPT_NAME) -ESCRIPT_SHEBANG ?= /usr/bin/env escript -ESCRIPT_STATIC ?= "deps/*/priv/**", "priv/**" - -# Core targets. - -distclean:: distclean-escript - -help:: - $(verbose) printf "%s\n" "" \ - "Escript targets:" \ - " escript Build an executable escript archive" \ - -# Plugin-specific targets. - -# Based on https://github.com/synrc/mad/blob/master/src/mad_bundle.erl -# Copyright (c) 2013 Maxim Sokhatsky, Synrc Research Center -# Modified MIT License, https://github.com/synrc/mad/blob/master/LICENSE : -# Software may only be used for the great good and the true happiness of all -# sentient beings. - -define ESCRIPT_RAW -'Read = fun(F) -> {ok, B} = file:read_file(filename:absname(F)), B end,'\ -'Files = fun(L) -> A = lists:concat([filelib:wildcard(X)||X<- L ]),'\ -' [F || F <- A, not filelib:is_dir(F) ] end,'\ -'Squash = fun(L) -> [{filename:basename(F), Read(F) } || F <- L ] end,'\ -'Zip = fun(A, L) -> {ok,{_,Z}} = zip:create(A, L, [{compress,all},memory]), Z end,'\ -'Ez = fun(Escript) ->'\ -' Static = Files([$(ESCRIPT_STATIC)]),'\ -' Beams = Squash(Files([$(ESCRIPT_BEAMS), $(ESCRIPT_SYS_CONFIG)])),'\ -' Archive = Beams ++ [{ "static.gz", Zip("static.gz", Static)}],'\ -' escript:create(Escript, [ $(ESCRIPT_OPTIONS)'\ -' {archive, Archive, [memory]},'\ -' {shebang, "$(ESCRIPT_SHEBANG)"},'\ -' {comment, "$(ESCRIPT_COMMENT)"},'\ -' {emu_args, " $(ESCRIPT_EMU_ARGS)"}'\ -' ]),'\ -' file:change_mode(Escript, 8#755)'\ -'end,'\ -'Ez("$(ESCRIPT_FILE)"),'\ -'halt().' -endef - -ESCRIPT_COMMAND = $(subst ' ',,$(ESCRIPT_RAW)) - -escript:: distclean-escript deps app - $(gen_verbose) $(ERL) -eval $(ESCRIPT_COMMAND) - -distclean-escript: - $(gen_verbose) rm -f $(ESCRIPT_NAME) - -# Copyright (c) 2014, Enrique Fernandez -# Copyright (c) 2015, Loïc Hoguin -# This file is contributed to erlang.mk and subject to the terms of the ISC License. - -.PHONY: eunit apps-eunit - -# Configuration - -EUNIT_OPTS ?= -EUNIT_ERL_OPTS ?= - -# Core targets. - -tests:: eunit - -help:: - $(verbose) printf "%s\n" "" \ - "EUnit targets:" \ - " eunit Run all the EUnit tests for this project" - -# Plugin-specific targets. - -define eunit.erl - case "$(COVER)" of - "" -> ok; - _ -> - case cover:compile_beam_directory("ebin") of - {error, _} -> halt(1); - _ -> ok - end - end, - case eunit:test($1, [$(EUNIT_OPTS)]) of - ok -> ok; - error -> halt(2) - end, - case "$(COVER)" of - "" -> ok; - _ -> - cover:export("eunit.coverdata") - end, - halt() -endef - -EUNIT_ERL_OPTS += -pa $(TEST_DIR) $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(CURDIR)/ebin - -ifdef t -ifeq (,$(findstring :,$(t))) -eunit: test-build - $(gen_verbose) $(call erlang,$(call eunit.erl,['$(t)']),$(EUNIT_ERL_OPTS)) -else -eunit: test-build - $(gen_verbose) $(call erlang,$(call eunit.erl,fun $(t)/0),$(EUNIT_ERL_OPTS)) -endif -else -EUNIT_EBIN_MODS = $(notdir $(basename $(ERL_FILES) $(BEAM_FILES))) -EUNIT_TEST_MODS = $(notdir $(basename $(call core_find,$(TEST_DIR)/,*.erl))) - -EUNIT_MODS = $(foreach mod,$(EUNIT_EBIN_MODS) $(filter-out \ - $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(EUNIT_TEST_MODS)),'$(mod)') - -eunit: test-build $(if $(IS_APP),,apps-eunit) - $(gen_verbose) $(call erlang,$(call eunit.erl,[$(call comma_list,$(EUNIT_MODS))]),$(EUNIT_ERL_OPTS)) - -ifneq ($(ALL_APPS_DIRS),) -apps-eunit: - $(verbose) for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1; done -endif -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: relx-rel distclean-relx-rel distclean-relx run - -# Configuration. - -RELX ?= $(CURDIR)/relx -RELX_CONFIG ?= $(CURDIR)/relx.config - -RELX_URL ?= https://github.com/erlware/relx/releases/download/v3.19.0/relx -RELX_OPTS ?= -RELX_OUTPUT_DIR ?= _rel - -ifeq ($(firstword $(RELX_OPTS)),-o) - RELX_OUTPUT_DIR = $(word 2,$(RELX_OPTS)) -else - RELX_OPTS += -o $(RELX_OUTPUT_DIR) -endif - -# Core targets. - -ifeq ($(IS_DEP),) -ifneq ($(wildcard $(RELX_CONFIG)),) -rel:: relx-rel -endif -endif - -distclean:: distclean-relx-rel distclean-relx - -# Plugin-specific targets. - -$(RELX): - $(gen_verbose) $(call core_http_get,$(RELX),$(RELX_URL)) - $(verbose) chmod +x $(RELX) - -relx-rel: $(RELX) rel-deps app - $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) - -distclean-relx-rel: - $(gen_verbose) rm -rf $(RELX_OUTPUT_DIR) - -distclean-relx: - $(gen_verbose) rm -rf $(RELX) - -# Run target. - -ifeq ($(wildcard $(RELX_CONFIG)),) -run: -else - -define get_relx_release.erl - {ok, Config} = file:consult("$(RELX_CONFIG)"), - {release, {Name, _}, _} = lists:keyfind(release, 1, Config), - io:format("~s", [Name]), - halt(0). -endef - -RELX_RELEASE = `$(call erlang,$(get_relx_release.erl))` - -run: all - $(verbose) $(RELX_OUTPUT_DIR)/$(RELX_RELEASE)/bin/$(RELX_RELEASE) console - -help:: - $(verbose) printf "%s\n" "" \ - "Relx targets:" \ - " run Compile the project, build the release and run it" - -endif - -# Copyright (c) 2014, M Robert Martin -# Copyright (c) 2015, Loïc Hoguin -# This file is contributed to erlang.mk and subject to the terms of the ISC License. - -.PHONY: shell - -# Configuration. - -SHELL_ERL ?= erl -SHELL_PATHS ?= $(CURDIR)/ebin $(APPS_DIR)/*/ebin $(DEPS_DIR)/*/ebin -SHELL_OPTS ?= - -ALL_SHELL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(SHELL_DEPS)) - -# Core targets - -help:: - $(verbose) printf "%s\n" "" \ - "Shell targets:" \ - " shell Run an erlang shell with SHELL_OPTS or reasonable default" - -# Plugin-specific targets. - -$(foreach dep,$(SHELL_DEPS),$(eval $(call dep_target,$(dep)))) - -build-shell-deps: $(ALL_SHELL_DEPS_DIRS) - $(verbose) for dep in $(ALL_SHELL_DEPS_DIRS) ; do $(MAKE) -C $$dep ; done - -shell: build-shell-deps - $(gen_verbose) $(SHELL_ERL) -pa $(SHELL_PATHS) $(SHELL_OPTS) - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -ifeq ($(filter triq,$(DEPS) $(TEST_DEPS)),triq) -.PHONY: triq - -# Targets. - -tests:: triq - -define triq_check.erl - code:add_pathsa(["$(CURDIR)/ebin", "$(DEPS_DIR)/*/ebin"]), - try - case $(1) of - all -> [true] =:= lists:usort([triq:check(M) || M <- [$(call comma_list,$(3))]]); - module -> triq:check($(2)); - function -> triq:check($(2)) - end - of - true -> halt(0); - _ -> halt(1) - catch error:undef -> - io:format("Undefined property or module~n"), - halt(0) - end. -endef - -ifdef t -ifeq (,$(findstring :,$(t))) -triq: test-build - $(verbose) $(call erlang,$(call triq_check.erl,module,$(t))) -else -triq: test-build - $(verbose) echo Testing $(t)/0 - $(verbose) $(call erlang,$(call triq_check.erl,function,$(t)())) -endif -else -triq: test-build - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename $(wildcard ebin/*.beam)))))) - $(gen_verbose) $(call erlang,$(call triq_check.erl,all,undefined,$(MODULES))) -endif -endif - -# Copyright (c) 2015, Erlang Solutions Ltd. -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: xref distclean-xref - -# Configuration. - -ifeq ($(XREF_CONFIG),) - XREF_ARGS := -else - XREF_ARGS := -c $(XREF_CONFIG) -endif - -XREFR ?= $(CURDIR)/xrefr -export XREFR - -XREFR_URL ?= https://github.com/inaka/xref_runner/releases/download/0.2.2/xrefr - -# Core targets. - -help:: - $(verbose) printf "%s\n" "" \ - "Xref targets:" \ - " xref Run Xrefr using $XREF_CONFIG as config file if defined" - -distclean:: distclean-xref - -# Plugin-specific targets. - -$(XREFR): - $(gen_verbose) $(call core_http_get,$(XREFR),$(XREFR_URL)) - $(verbose) chmod +x $(XREFR) - -xref: deps app $(XREFR) - $(gen_verbose) $(XREFR) $(XREFR_ARGS) - -distclean-xref: - $(gen_verbose) rm -rf $(XREFR) - -# Copyright 2015, Viktor Söderqvist -# This file is part of erlang.mk and subject to the terms of the ISC License. - -COVER_REPORT_DIR = cover - -# Hook in coverage to ct - -ifdef COVER -ifdef CT_RUN -# All modules in 'ebin' -COVER_MODS = $(notdir $(basename $(call core_ls,ebin/*.beam))) - -test-build:: $(TEST_DIR)/ct.cover.spec - -$(TEST_DIR)/ct.cover.spec: - $(verbose) echo Cover mods: $(COVER_MODS) - $(gen_verbose) printf "%s\n" \ - '{incl_mods,[$(subst $(space),$(comma),$(COVER_MODS))]}.' \ - '{export,"$(CURDIR)/ct.coverdata"}.' > $@ - -CT_RUN += -cover $(TEST_DIR)/ct.cover.spec -endif -endif - -# Core targets - -ifdef COVER -ifneq ($(COVER_REPORT_DIR),) -tests:: - $(verbose) $(MAKE) --no-print-directory cover-report -endif -endif - -clean:: coverdata-clean - -ifneq ($(COVER_REPORT_DIR),) -distclean:: cover-report-clean -endif - -help:: - $(verbose) printf "%s\n" "" \ - "Cover targets:" \ - " cover-report Generate a HTML coverage report from previously collected" \ - " cover data." \ - " all.coverdata Merge {eunit,ct}.coverdata into one coverdata file." \ - "" \ - "If COVER=1 is set, coverage data is generated by the targets eunit and ct. The" \ - "target tests additionally generates a HTML coverage report from the combined" \ - "coverdata files from each of these testing tools. HTML reports can be disabled" \ - "by setting COVER_REPORT_DIR to empty." - -# Plugin specific targets - -COVERDATA = $(filter-out all.coverdata,$(wildcard *.coverdata)) - -.PHONY: coverdata-clean -coverdata-clean: - $(gen_verbose) rm -f *.coverdata ct.cover.spec - -# Merge all coverdata files into one. -all.coverdata: $(COVERDATA) - $(gen_verbose) $(ERL) -eval ' \ - $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \ - cover:export("$@"), halt(0).' - -# These are only defined if COVER_REPORT_DIR is non-empty. Set COVER_REPORT_DIR to -# empty if you want the coverdata files but not the HTML report. -ifneq ($(COVER_REPORT_DIR),) - -.PHONY: cover-report-clean cover-report - -cover-report-clean: - $(gen_verbose) rm -rf $(COVER_REPORT_DIR) - -ifeq ($(COVERDATA),) -cover-report: -else - -# Modules which include eunit.hrl always contain one line without coverage -# because eunit defines test/0 which is never called. We compensate for this. -EUNIT_HRL_MODS = $(subst $(space),$(comma),$(shell \ - grep -e '^\s*-include.*include/eunit\.hrl"' src/*.erl \ - | sed "s/^src\/\(.*\)\.erl:.*/'\1'/" | uniq)) - -define cover_report.erl - $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) - Ms = cover:imported_modules(), - [cover:analyse_to_file(M, "$(COVER_REPORT_DIR)/" ++ atom_to_list(M) - ++ ".COVER.html", [html]) || M <- Ms], - Report = [begin {ok, R} = cover:analyse(M, module), R end || M <- Ms], - EunitHrlMods = [$(EUNIT_HRL_MODS)], - Report1 = [{M, {Y, case lists:member(M, EunitHrlMods) of - true -> N - 1; false -> N end}} || {M, {Y, N}} <- Report], - TotalY = lists:sum([Y || {_, {Y, _}} <- Report1]), - TotalN = lists:sum([N || {_, {_, N}} <- Report1]), - Perc = fun(Y, N) -> case Y + N of 0 -> 100; S -> round(100 * Y / S) end end, - TotalPerc = Perc(TotalY, TotalN), - {ok, F} = file:open("$(COVER_REPORT_DIR)/index.html", [write]), - io:format(F, "~n" - "~n" - "Coverage report~n" - "~n", []), - io:format(F, "

Coverage

~n

Total: ~p%

~n", [TotalPerc]), - io:format(F, "~n", []), - [io:format(F, "" - "~n", - [M, M, Perc(Y, N)]) || {M, {Y, N}} <- Report1], - How = "$(subst $(space),$(comma)$(space),$(basename $(COVERDATA)))", - Date = "$(shell date -u "+%Y-%m-%dT%H:%M:%SZ")", - io:format(F, "
ModuleCoverage
~p~p%
~n" - "

Generated using ~s and erlang.mk on ~s.

~n" - "", [How, Date]), - halt(). -endef - -cover-report: - $(gen_verbose) mkdir -p $(COVER_REPORT_DIR) - $(gen_verbose) $(call erlang,$(cover_report.erl)) - -endif -endif # ifneq ($(COVER_REPORT_DIR),) From 64f62fa0ce5095a75f77792daee2dc646bfea524 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 4 Dec 2018 23:22:39 +0800 Subject: [PATCH 14/54] Make some processes hibernate after 1s. --- src/emqx_broker.erl | 2 +- src/emqx_hooks.erl | 2 +- src/emqx_pool.erl | 3 ++- src/emqx_router.erl | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index ca4a86c87..b816a0a69 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -56,7 +56,7 @@ -spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). start_link(Pool, Id) -> gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, - [Pool, Id], [{hibernate_after, 2000}]). + [Pool, Id], [{hibernate_after, 1000}]). %%------------------------------------------------------------------------------ %% Subscribe diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl index 073c12870..b10445742 100644 --- a/src/emqx_hooks.erl +++ b/src/emqx_hooks.erl @@ -42,7 +42,7 @@ -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], [{hibernate_after, 60000}]). + gen_server:start_link({local, ?SERVER}, ?MODULE, [], [{hibernate_after, 1000}]). -spec(stop() -> ok). stop() -> diff --git a/src/emqx_pool.erl b/src/emqx_pool.erl index 762f5dc6d..7b12bea69 100644 --- a/src/emqx_pool.erl +++ b/src/emqx_pool.erl @@ -35,7 +35,8 @@ start_link() -> %% @doc Start pool. -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], []). + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). %% @doc Submit work to the pool. -spec(submit(task()) -> any()). diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 941c004f7..e513d041d 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -68,7 +68,7 @@ mnesia(copy) -> -spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). start_link(Pool, Id) -> gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, - ?MODULE, [Pool, Id], [{hibernate_after, 2000}]). + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). %%------------------------------------------------------------------------------ %% Route APIs From cca5081e021c509dbdba07bb9037e9321f8cf546 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Mon, 10 Dec 2018 18:37:42 +0800 Subject: [PATCH 15/54] Improve the design of trie, router and broker modules 1. Add do_add_route/1 do_add_route/2, do_delete_route/1, do_delete_route/2 APIs in emqx_router module 2. Improve the code of emqx_trie module 3. Update the emqx_broker module to call the new APIs of emqx_router --- src/emqx_broker.erl | 30 ++++----- src/emqx_router.erl | 134 ++++++++++++++++++------------------- src/emqx_router_helper.erl | 45 +++++++------ src/emqx_shared_sub.erl | 7 +- src/emqx_trie.erl | 18 ++--- test/emqx_router_SUITE.erl | 103 ++++++++++------------------ test/emqx_trie_SUITE.erl | 113 ++++++++++++++++--------------- 7 files changed, 208 insertions(+), 242 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index a2ddf5b60..a00dc17b8 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -27,6 +27,7 @@ -export([subscriptions/1, subscribers/1, subscribed/2]). -export([get_subopts/2, set_subopts/2]). -export([topics/0]). + %% Stats fun -export([stats_fun/0]). @@ -52,9 +53,9 @@ -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - _ = create_tabs(), - Name = emqx_misc:proc_name(?BROKER, Id), - gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], []). + ok = create_tabs(), + gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, + ?MODULE, [Pool, Id], []). %%------------------------------------------------------------------------------ %% Create tabs @@ -75,7 +76,7 @@ create_tabs() -> ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... - %% duplicate_bag: o(1) insert + %% bag: o(n) insert ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ @@ -114,7 +115,7 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map call(pick({Topic, Shard}), {subscribe, Topic}); Group -> %% Shard subscription true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - emqx_shard_sub:subscribe(Group, Topic, SubPid) + emqx_shared_sub:subscribe(Group, Topic, SubPid) end; true -> ok end. @@ -367,18 +368,17 @@ pick(Topic) -> %%------------------------------------------------------------------------------ init([Pool, Id]) -> - _ = emqx_router:set_mode(protected), true = gproc_pool:connect_worker(Pool, {Pool, Id}), {ok, #{pool => Pool, id => Id}}. handle_call({subscribe, Topic}, _From, State) -> - case get(Topic) of - undefined -> - _ = put(Topic, true), - emqx_router:add_route(Topic); - true -> ok - end, - {reply, ok, State}; + Ok = case get(Topic) of + undefined -> + _ = put(Topic, true), + emqx_router:do_add_route(Topic); + true -> ok + end, + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), @@ -387,8 +387,8 @@ handle_call(Req, _From, State) -> handle_cast({unsubscribed, Topic}, State) -> case ets:member(?SUBSCRIBER, Topic) of false -> - _ = erase(Topic), - emqx_router:delete_route(Topic); + _ = erase(Topic), + emqx_router:do_delete_route(Topic); true -> ok end, {noreply, State}; diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 313adc475..c165b97ca 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -29,19 +29,19 @@ %% Route APIs -export([add_route/1, add_route/2]). --export([get_routes/1]). +-export([do_add_route/1, do_add_route/2]). +-export([match_routes/1, lookup_routes/1, has_routes/1]). -export([delete_route/1, delete_route/2]). --export([has_routes/1, match_routes/1, print_routes/1]). +-export([do_delete_route/1, do_delete_route/2]). +-export([print_routes/1]). -export([topics/0]). -%% Mode --export([set_mode/1, get_mode/0]). - %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --type(destination() :: node() | {binary(), node()}). +-type(group() :: binary()). +-type(destination() :: node() | {group(), node()}). -define(ROUTE, emqx_route). @@ -66,75 +66,76 @@ mnesia(copy) -> -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - Name = emqx_misc:proc_name(?MODULE, Id), - gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). %%------------------------------------------------------------------------------ %% Route APIs %%------------------------------------------------------------------------------ --spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). +-spec(add_route(emqx_topic:topic()) -> ok | {error, term()}). add_route(Topic) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = node()}); -add_route(Route = #route{topic = Topic}) -> - case get_mode() of - protected -> do_add_route(Route); - undefined -> call(pick(Topic), {add_route, Route}) - end. + add_route(Topic, node()). -spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). add_route(Topic, Dest) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = Dest}). + call(pick(Topic), {add_route, Topic, Dest}). -%% @private -do_add_route(Route = #route{topic = Topic, dest = Dest}) -> - case lists:member(Route, get_routes(Topic)) of +-spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}). +do_add_route(Topic) when is_binary(Topic) -> + do_add_route(Topic, node()). + +-spec(do_add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_add_route(Topic, Dest) when is_binary(Topic) -> + Route = #route{topic = Topic, dest = Dest}, + case lists:member(Route, lookup_routes(Topic)) of true -> ok; false -> ok = emqx_router_helper:monitor(Dest), case emqx_topic:wildcard(Topic) of - true -> trans(fun add_trie_route/1, [Route]); - false -> add_direct_route(Route) + true -> trans(fun insert_trie_route/1, [Route]); + false -> insert_direct_route(Route) end end. --spec(get_routes(emqx_topic:topic()) -> [emqx_types:route()]). -get_routes(Topic) -> +%% @doc Match routes +-spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). +match_routes(Topic) when is_binary(Topic) -> + %% Optimize: routing table will be replicated to all router nodes. + Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), + lists:append([lookup_routes(To) || To <- [Topic | Matched]]). + +-spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). +lookup_routes(Topic) -> ets:lookup(?ROUTE, Topic). --spec(delete_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). -delete_route(Topic) when is_binary(Topic) -> - delete_route(#route{topic = Topic, dest = node()}); -delete_route(Route = #route{topic = Topic}) -> - case get_mode() of - protected -> do_delete_route(Route); - undefined -> call(pick(Topic), {delete_route, Route}) - end. - --spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). -delete_route(Topic, Dest) when is_binary(Topic) -> - delete_route(#route{topic = Topic, dest = Dest}). - -%% @private -do_delete_route(Route = #route{topic = Topic}) -> - case emqx_topic:wildcard(Topic) of - true -> trans(fun del_trie_route/1, [Route]); - false -> del_direct_route(Route) - end. - -spec(has_routes(emqx_topic:topic()) -> boolean()). has_routes(Topic) when is_binary(Topic) -> ets:member(?ROUTE, Topic). --spec(topics() -> list(emqx_topic:topic())). -topics() -> mnesia:dirty_all_keys(?ROUTE). +-spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}). +delete_route(Topic) when is_binary(Topic) -> + delete_route(Topic, node()). -%% @doc Match routes -%% Optimize: routing table will be replicated to all router nodes. --spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). -match_routes(Topic) when is_binary(Topic) -> - Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), - lists:append([get_routes(To) || To <- [Topic | Matched]]). +-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +delete_route(Topic, Dest) when is_binary(Topic) -> + call(pick(Topic), {delete_route, Topic, Dest}). + +-spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}). +do_delete_route(Topic) when is_binary(Topic) -> + do_delete_route(Topic, node()). + +-spec(do_delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_delete_route(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> trans(fun delete_trie_route/1, [Route]); + false -> delete_direct_route(Route) + end. + +-spec(topics() -> list(emqx_topic:topic())). +topics() -> + mnesia:dirty_all_keys(?ROUTE). %% @doc Print routes to a topic -spec(print_routes(emqx_topic:topic()) -> ok). @@ -143,13 +144,6 @@ print_routes(Topic) -> io:format("~s -> ~s~n", [To, Dest]) end, match_routes(Topic)). --spec(set_mode(protected | atom()) -> any()). -set_mode(Mode) when is_atom(Mode) -> - put('$router_mode', Mode). - --spec(get_mode() -> protected | undefined | atom()). -get_mode() -> get('$router_mode'). - call(Router, Msg) -> gen_server:call(Router, Msg, infinity). @@ -164,11 +158,13 @@ init([Pool, Id]) -> true = gproc_pool:connect_worker(Pool, {Pool, Id}), {ok, #{pool => Pool, id => Id}}. -handle_call({add_route, Route}, _From, State) -> - {reply, do_add_route(Route), State}; +handle_call({add_route, Topic, Dest}, _From, State) -> + Ok = do_add_route(Topic, Dest), + {reply, Ok, State}; -handle_call({delete_route, Route}, _From, State) -> - {reply, do_delete_route(Route), State}; +handle_call({delete_route, Topic, Dest}, _From, State) -> + Ok = do_delete_route(Topic, Dest), + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Router] unexpected call: ~p", [Req]), @@ -192,23 +188,23 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -add_direct_route(Route) -> +insert_direct_route(Route) -> mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]). -add_trie_route(Route = #route{topic = Topic}) -> +insert_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [] -> emqx_trie:insert(Topic); _ -> ok end, mnesia:write(?ROUTE, Route, sticky_write). -del_direct_route(Route) -> +delete_direct_route(Route) -> mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]). -del_trie_route(Route = #route{topic = Topic}) -> +delete_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [Route] -> %% Remove route and trie - mnesia:delete_object(?ROUTE, Route, sticky_write), + ok = mnesia:delete_object(?ROUTE, Route, sticky_write), emqx_trie:delete(Topic); [_|_] -> %% Remove route only mnesia:delete_object(?ROUTE, Route, sticky_write); @@ -219,7 +215,7 @@ del_trie_route(Route = #route{topic = Topic}) -> -spec(trans(function(), list(any())) -> ok | {error, term()}). trans(Fun, Args) -> case mnesia:transaction(Fun, Args) of - {atomic, _} -> ok; - {aborted, Error} -> {error, Error} + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} end. diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index c24b10715..efeaabc74 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -31,15 +31,11 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -%% internal export +%% Internal export -export([stats_fun/0]). -record(routing_node, {name, const = unused}). --record(state, {nodes = []}). --compile({no_auto_import, [monitor/1]}). - --define(SERVER, ?MODULE). -define(ROUTE, emqx_route). -define(ROUTING_NODE, emqx_routing_node). -define(LOCK, {?MODULE, cleanup_routes}). @@ -64,9 +60,9 @@ mnesia(copy) -> %%------------------------------------------------------------------------------ %% @doc Starts the router helper --spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). %% @doc Monitor routing node -spec(monitor(node() | {binary(), node()}) -> ok). @@ -84,18 +80,18 @@ monitor(Node) when is_atom(Node) -> %%------------------------------------------------------------------------------ init([]) -> - _ = ekka:monitor(membership), - _ = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + ok = ekka:monitor(membership), + {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of true -> Acc; - false -> _ = erlang:monitor_node(Node, true), + false -> true = erlang:monitor_node(Node, true), [Node | Acc] end end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), - {ok, #state{nodes = Nodes}, hibernate}. + {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> emqx_logger:error("[RouterHelper] unexpected call: ~p", [Req]), @@ -105,24 +101,29 @@ handle_cast(Msg, State) -> emqx_logger:error("[RouterHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({mnesia_table_event, {write, #routing_node{name = Node}, _}}, State = #state{nodes = Nodes}) -> - emqx_logger:info("[RouterHelper] write routing node: ~s", [Node]), +handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, State = #{nodes := Nodes}) -> case ekka:is_member(Node) orelse lists:member(Node, Nodes) of - true -> {noreply, State}; - false -> _ = erlang:monitor_node(Node, true), - {noreply, State#state{nodes = [Node | Nodes]}} + true -> {noreply, State}; + false -> + true = erlang:monitor_node(Node, true), + {noreply, State#{nodes := [Node | Nodes]}} end; -handle_info({mnesia_table_event, _Event}, State) -> +handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> + %% ignore {noreply, State}; -handle_info({nodedown, Node}, State = #state{nodes = Nodes}) -> +handle_info({mnesia_table_event, Event}, State) -> + emqx_logger:error("[RouterHelper] unexpected mnesia_table_event: ~p", [Event]), + {noreply, State}; + +handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> global:trans({?LOCK, self()}, fun() -> mnesia:transaction(fun cleanup_routes/1, [Node]) end), - mnesia:dirty_delete(?ROUTING_NODE, Node), - {noreply, State#state{nodes = lists:delete(Node, Nodes)}, hibernate}; + ok = mnesia:dirty_delete(?ROUTING_NODE, Node), + {noreply, State#{nodes := lists:delete(Node, Nodes)}, hibernate}; handle_info({membership, {mnesia, down, Node}}, State) -> handle_info({nodedown, Node}, State); @@ -134,8 +135,8 @@ handle_info(Info, State) -> emqx_logger:error("[RouteHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> - ekka:unmonitor(membership), +terminate(_Reason, _State) -> + ok = ekka:unmonitor(membership), emqx_stats:cancel_update(route_stats), mnesia:unsubscribe({table, ?ROUTING_NODE, simple}). diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index d1d0d921d..b7e41213b 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -252,7 +252,6 @@ subscribers(Group, Topic) -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_router:set_mode(protected), mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mnesia:transaction(fun init_monitors/0), ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), @@ -269,7 +268,7 @@ handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:add_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) end, ok = maybe_insert_alive_tab(SubPid), true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), @@ -280,7 +279,7 @@ handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end, {reply, ok, State}; @@ -334,7 +333,7 @@ cleanup_down(SubPid) -> true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 79f6042b7..27ff52827 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -36,7 +36,7 @@ %% @doc Create or replicate trie tables. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> - %% Optimize + %% Optimize storage StoreProps = [{ets, [{read_concurrency, true}, {write_concurrency, true}]}], %% Trie table @@ -72,7 +72,7 @@ insert(Topic) when is_binary(Topic) -> write_trie_node(TrieNode#trie_node{topic = Topic}); [] -> %% Add trie path - lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), + ok = lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), %% Add last node write_trie_node(#trie_node{node_id = Topic, topic = Topic}) end. @@ -93,7 +93,7 @@ lookup(NodeId) -> delete(Topic) when is_binary(Topic) -> case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{edge_count = 0}] -> - mnesia:delete({?TRIE_NODE, Topic}), + ok = mnesia:delete({?TRIE_NODE, Topic}), delete_path(lists:reverse(emqx_topic:triples(Topic))); [TrieNode] -> write_trie_node(TrieNode#trie_node{topic = undefined}); @@ -112,12 +112,12 @@ add_path({Node, Word, Child}) -> [TrieNode = #trie_node{edge_count = Count}] -> case mnesia:wread({?TRIE, Edge}) of [] -> - write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), + ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), write_trie(#trie{edge = Edge, node_id = Child}); [_] -> ok end; [] -> - write_trie_node(#trie_node{node_id = Node, edge_count = 1}), + ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}), write_trie(#trie{edge = Edge, node_id = Child}) end. @@ -154,10 +154,10 @@ match_node(NodeId, [W|Words], ResAcc) -> delete_path([]) -> ok; delete_path([{NodeId, Word, _} | RestPath]) -> - mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), - case mnesia:read(?TRIE_NODE, NodeId) of + ok = mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), + case mnesia:wread({?TRIE_NODE, NodeId}) of [#trie_node{edge_count = 1, topic = undefined}] -> - mnesia:delete({?TRIE_NODE, NodeId}), + ok = mnesia:delete({?TRIE_NODE, NodeId}), delete_path(RestPath); [TrieNode = #trie_node{edge_count = 1, topic = _}] -> write_trie_node(TrieNode#trie_node{edge_count = 0}); @@ -167,9 +167,11 @@ delete_path([{NodeId, Word, _} | RestPath]) -> mnesia:abort({node_not_found, NodeId}) end. +%% @private write_trie(Trie) -> mnesia:write(?TRIE, Trie, write). +%% @private write_trie_node(TrieNode) -> mnesia:write(?TRIE_NODE, TrieNode, write). diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl index e317ec7b3..c115fd0cd 100644 --- a/test/emqx_router_SUITE.erl +++ b/test/emqx_router_SUITE.erl @@ -21,17 +21,16 @@ -compile(nowarn_export_all). -define(R, emqx_router). --define(TABS, [emqx_route, emqx_trie, emqx_trie_node]). all() -> [{group, route}]. groups() -> [{route, [sequence], - [add_del_route, - match_routes, - has_routes, - router_add_del]}]. + [t_add_delete, + t_do_add_delete, + t_match_routes, + t_has_routes]}]. init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), @@ -47,77 +46,47 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). -add_del_route(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/+/b">>, node()), - ct:log("Topics: ~p ~n", [emqx_topic:wildcard(<<"a/+/b">>)]), - timer:sleep(1), - +t_add_delete(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/b">>, node()), ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), - ?R:del_route(From, <<"a/b/c">>, node()), + ?R:delete_route(<<"a/b/c">>), + ?R:delete_route(<<"a/+/b">>, node()), + ?assertEqual([], ?R:topics()). - ?R:del_route(From, <<"a/+/b">>, node()), - timer:sleep(120), - ?assertEqual([], lists:sort(?R:topics())). +t_do_add_delete(_) -> + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/+/b">>, node()), + ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), -match_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - ?R:add_route(From, <<"a/+/c">>, node()), - ?R:add_route(From, <<"a/b/#">>, node()), - ?R:add_route(From, <<"#">>, node()), - timer:sleep(1000), + ?R:do_delete_route(<<"a/b/c">>, node()), + ?R:do_delete_route(<<"a/+/b">>), + ?assertEqual([], ?R:topics()). + +t_match_routes(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/c">>, node()), + ?R:add_route(<<"a/b/#">>, node()), + ?R:add_route(<<"#">>, node()), ?assertEqual([#route{topic = <<"#">>, dest = node()}, #route{topic = <<"a/+/c">>, dest = node()}, #route{topic = <<"a/b/#">>, dest = node()}, #route{topic = <<"a/b/c">>, dest = node()}], - lists:sort(?R:match_routes(<<"a/b/c">>))). + lists:sort(?R:match_routes(<<"a/b/c">>))), + ?R:delete_route(<<"a/b/c">>, node()), + ?R:delete_route(<<"a/+/c">>, node()), + ?R:delete_route(<<"a/b/#">>, node()), + ?R:delete_route(<<"#">>, node()), + ?assertEqual([], lists:sort(?R:match_routes(<<"a/b/c">>))). -has_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"devices/+/messages">>, node()), - timer:sleep(200), - ?assert(?R:has_routes(<<"devices/+/messages">>)). +t_has_routes(_) -> + ?R:add_route(<<"devices/+/messages">>, node()), + ?assert(?R:has_routes(<<"devices/+/messages">>)), + ?R:delete_route(<<"devices/+/messages">>). clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, ?TABS). - -router_add_del(_) -> - ?R:add_route(<<"#">>), - ?R:add_route(<<"a/b/c">>, node()), - ?R:add_route(<<"+/#">>), - Routes = [R1, R2 | _] = [ - #route{topic = <<"#">>, dest = node()}, - #route{topic = <<"+/#">>, dest = node()}, - #route{topic = <<"a/b/c">>, dest = node()}], - timer:sleep(500), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - ?R:print_routes(<<"a/b/c">>), - - %% Batch Add - lists:foreach(fun(R) -> ?R:add_route(R) end, Routes), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - %% Del - ?R:del_route(<<"a/b/c">>, node()), - timer:sleep(500), - [R1, R2] = lists:sort(?R:match_routes(<<"a/b/c">>)), - {atomic, []} = mnesia:transaction(fun emqx_trie:lookup/1, [<<"a/b/c">>]), - - %% Batch Del - R3 = #route{topic = <<"#">>, dest = 'a@127.0.0.1'}, - ?R:add_route(R3), - ?R:del_route(<<"#">>), - ?R:del_route(R2), - ?R:del_route(R3), - timer:sleep(500), - [] = lists:sort(?R:match_routes(<<"a/b/c">>)). + lists:foreach(fun mnesia:clear_table/1, [emqx_route, emqx_trie, emqx_trie_node]). diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 85637a447..09226979f 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -47,36 +47,36 @@ t_insert(_) -> edge_count = 3, topic = <<"sensor">>, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:lookup(<<"sensor">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:lookup(<<"sensor">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_match(_) -> Machted = [<<"sensor/+/#">>, <<"sensor/#">>], - {atomic, Machted} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:match(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:match(<<"sensor/1">>) + end, + ?assertEqual({atomic, Machted}, mnesia:transaction(Fun)). t_match2(_) -> Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, - {atomic, Matched} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"#">>), - ?TRIE:insert(<<"+/#">>), - ?TRIE:insert(<<"+/+/#">>), - {?TRIE:match(<<"a/b/c">>), - ?TRIE:match(<<"$SYS/broker/zenmq">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"#">>), + ?TRIE:insert(<<"+/#">>), + ?TRIE:insert(<<"+/+/#">>), + {?TRIE:match(<<"a/b/c">>), + ?TRIE:match(<<"$SYS/broker/zenmq">>)} + end, + ?assertEqual({atomic, Matched}, mnesia:transaction(Fun)). t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], @@ -91,43 +91,42 @@ t_delete(_) -> edge_count = 2, topic = undefined, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/#">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:lookup(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/#">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:lookup(<<"sensor/1">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_delete2(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric/3">>), - {?TRIE:lookup(<<"sensor">>), - ?TRIE:lookup(<<"sensor/1">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric/3">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). t_delete3(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/+">>), - ?TRIE:insert(<<"sensor/+/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor/+/metric/2">>), - ?TRIE:delete(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/+">>), - ?TRIE:delete(<<"sensor/+/unknown">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/+">>), + ?TRIE:insert(<<"sensor/+/metric/2">>), + ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor/+/metric/2">>), + ?TRIE:delete(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/+">>), + ?TRIE:delete(<<"sensor/+/unknown">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). clear_tables() -> lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS). From 7074707d6459a7698c490c044439adbfb7b1da16 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 11 Dec 2018 14:06:23 +0800 Subject: [PATCH 16/54] Add t_mnesia/1 test case --- test/emqx_trie_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 09226979f..500fe3574 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -24,7 +24,7 @@ -define(TRIE_TABS, [emqx_trie, emqx_trie_node]). all() -> - [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. + [t_mnesia, t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. init_per_suite(Config) -> application:load(emqx), @@ -42,6 +42,9 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). +t_mnesia(_) -> + ok = ?TRIE:mnesia(copy). + t_insert(_) -> TN = #trie_node{node_id = <<"sensor">>, edge_count = 3, From 47e3cd3692722400153e95463cca8d8532615607 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 13:34:13 +0800 Subject: [PATCH 17/54] Improve the subscription sharding. --- src/emqx.erl | 6 +- src/emqx_broker.erl | 217 +++++++++++++++++++---------------- src/emqx_broker_helper.erl | 80 ++++++++----- src/emqx_router_helper.erl | 2 +- src/emqx_sequence.erl | 1 + src/emqx_sm.erl | 2 +- src/emqx_tables.erl | 14 +++ test/emqx_sequence_SUITE.erl | 3 +- test/emqx_tables_SUITE.erl | 8 +- 9 files changed, 197 insertions(+), 136 deletions(-) diff --git a/src/emqx.erl b/src/emqx.erl index 3792cc4f8..76e966a59 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -22,7 +22,7 @@ %% PubSub API -export([subscribe/1, subscribe/2, subscribe/3]). -export([publish/1]). --export([unsubscribe/1, unsubscribe/2]). +-export([unsubscribe/1]). %% PubSub management API -export([topics/0, subscriptions/1, subscribers/1, subscribed/2]). @@ -88,10 +88,6 @@ publish(Msg) -> unsubscribe(Topic) -> emqx_broker:unsubscribe(iolist_to_binary(Topic)). --spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid()) -> ok). -unsubscribe(Topic, SubId) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId). - %%------------------------------------------------------------------------------ %% PubSub management API %%------------------------------------------------------------------------------ diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index a00dc17b8..9ed4aad06 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -20,7 +20,7 @@ -export([start_link/2]). -export([subscribe/1, subscribe/2, subscribe/3]). --export([unsubscribe/1, unsubscribe/2]). +-export([unsubscribe/1]). -export([subscriber_down/1]). -export([publish/1, safe_publish/1]). -export([dispatch/2]). @@ -35,6 +35,8 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-import(emqx_tables, [lookup_value/2, lookup_value/3]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -42,8 +44,7 @@ -define(BROKER, ?MODULE). -%% ETS tables --define(SUBID, emqx_subid). +%% ETS tables for PubSub -define(SUBOPTION, emqx_suboption). -define(SUBSCRIBER, emqx_subscriber). -define(SUBSCRIPTION, emqx_subscription). @@ -65,9 +66,6 @@ start_link(Pool, Id) -> create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], - %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [set | TabOpts]), - %% SubOption: {SubPid, Topic} -> SubOption ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]), @@ -76,7 +74,7 @@ create_tabs() -> ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... - %% bag: o(n) insert + %% bag: o(n) insert:( ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ @@ -98,28 +96,37 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map SubPid = self(), case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> - ok = emqx_broker_helper:monitor(SubPid, SubId), - %% true = ets:insert(?SUBID, {SubId, SubPid}), - true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), - case maps:get(share, SubOpts, undefined) of - undefined -> - Shard = emqx_broker_helper:get_shard(SubPid, Topic), - case Shard of - 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}); - I -> - true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}) - end, - SubOpts1 = maps:put(shard, Shard, SubOpts), - true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts1}), - call(pick({Topic, Shard}), {subscribe, Topic}); - Group -> %% Shard subscription - true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - emqx_shared_sub:subscribe(Group, Topic, SubPid) - end; + ok = emqx_broker_helper:monitor_sub(SubPid, SubId), + do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); true -> ok end. +with_subid(undefined, SubOpts) -> + SubOpts; +with_subid(SubId, SubOpts) -> + maps:put(subid, SubId, SubOpts). + +%% @private +do_subscribe(Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_subscribe(Group, Topic, SubPid, SubOpts). + +do_subscribe(undefined, Topic, SubPid, SubOpts) -> + case emqx_broker_helper:get_sub_shard(SubPid, Topic) of + 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + call(pick(Topic), {subscribe, Topic}); + I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, maps:put(shard, I, SubOpts)}), + call(pick({Topic, I}), {subscribe, Topic, I}) + end; + +%% Shared subscription +do_subscribe(Group, Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + emqx_shared_sub:subscribe(Group, Topic, SubPid). + %%------------------------------------------------------------------------------ %% Unsubscribe API %%------------------------------------------------------------------------------ @@ -130,33 +137,26 @@ unsubscribe(Topic) when is_binary(Topic) -> case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> _ = emqx_broker_helper:reclaim_seq(Topic), - case maps:get(share, SubOpts, undefined) of - undefined -> - case maps:get(shard, SubOpts, 0) of - 0 -> - true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> - true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shard, Topic, I}) of - true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) - end, - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; - Group -> - ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid) - end, - true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), - %%true = ets:delete_object(?SUBID, {SubId, SubPid}), - true = ets:delete(?SUBOPTION, {SubPid, Topic}), - ok; + do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok end. --spec(unsubscribe(emqx_topic:topic(), emqx_types:subid()) -> ok). -unsubscribe(Topic, _SubId) when is_binary(Topic) -> - unsubscribe(Topic). +do_unsubscribe(Topic, SubPid, SubOpts) -> + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_unsubscribe(Group, Topic, SubPid, SubOpts). + +do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + +do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> + emqx_shared_sub:unsubscribe(Group, Topic, SubPid). %%------------------------------------------------------------------------------ %% Publish @@ -241,23 +241,28 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> inc_dropped_cnt(Topic), Delivery; [Sub] -> %% optimize? - dispatch(Sub, Topic, Msg), - Delivery#delivery{results = [{dispatch, Topic, 1}|Results]}; + Cnt = dispatch(Sub, Topic, Msg), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]}; Subs -> - Count = lists:foldl( - fun(Sub, Acc) -> - dispatch(Sub, Topic, Msg), Acc + 1 - end, 0, Subs), - Delivery#delivery{results = [{dispatch, Topic, Count}|Results]} + Cnt = lists:foldl( + fun(Sub, Acc) -> + dispatch(Sub, Topic, Msg) + Acc + end, 0, Subs), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]} end. dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; + case erlang:is_process_alive(SubPid) of + true -> + SubPid ! {dispatch, Topic, Msg}, + 1; + false -> 0 + end; dispatch({shard, I}, Topic, Msg) -> - - lists:foreach(fun(SubPid) -> - SubPid ! {dispatch, Topic, Msg} - end, safe_lookup_element(?SUBSCRIBER, {shard, Topic, I}, [])). + lists:foldl( + fun(SubPid, Cnt) -> + dispatch(SubPid, Topic, Msg) + Cnt + end, 0, subscribers({shard, Topic, I})). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; @@ -265,8 +270,10 @@ inc_dropped_cnt(_Topic) -> emqx_metrics:inc('messages/dropped'). -spec(subscribers(emqx_topic:topic()) -> [pid()]). -subscribers(Topic) -> - safe_lookup_element(?SUBSCRIBER, Topic, []). +subscribers(Topic) when is_binary(Topic) -> + lookup_value(?SUBSCRIBER, Topic, []); +subscribers(Shard = {shard, _Topic, _I}) -> + lookup_value(?SUBSCRIBER, Shard, []). %%------------------------------------------------------------------------------ %% Subscriber is down @@ -275,27 +282,21 @@ subscribers(Topic) -> -spec(subscriber_down(pid()) -> true). subscriber_down(SubPid) -> lists:foreach( - fun(Sub = {_Pid, Topic}) -> - case ets:lookup(?SUBOPTION, Sub) of - [{_, SubOpts}] -> + fun(Topic) -> + case lookup_value(?SUBOPTION, {SubPid, Topic}) of + SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), + true = ets:delete(?SUBOPTION, {SubPid, Topic}), case maps:get(shard, SubOpts, 0) of - 0 -> - true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> - true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shard, Topic, I}) of - true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) - end, - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end, - ets:delete(?SUBOPTION, Sub); - [] -> ok + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + undefined -> ok end - end, ets:lookup(?SUBSCRIPTION, SubPid)), - true = ets:delete(?SUBSCRIPTION, SubPid). + end, lookup_value(?SUBSCRIPTION, SubPid, [])), + ets:delete(?SUBSCRIPTION, SubPid). %%------------------------------------------------------------------------------ %% Management APIs @@ -303,20 +304,32 @@ subscriber_down(SubPid) -> -spec(subscriptions(pid() | emqx_types:subid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(SubPid) -> - [{Topic, safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{})} - || Topic <- safe_lookup_element(?SUBSCRIPTION, SubPid, [])]. +subscriptions(SubPid) when is_pid(SubPid) -> + [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})} + || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])]; +subscriptions(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + subscriptions(SubPid); + undefined -> [] + end. -spec(subscribed(pid(), emqx_topic:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); subscribed(SubId, Topic) when ?is_subid(SubId) -> - %%FIXME:... SubId -> SubPid - ets:member(?SUBOPTION, {SubId, Topic}). + SubPid = emqx_broker_helper:lookup_subpid(SubId), + ets:member(?SUBOPTION, {SubPid, Topic}). --spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts()). +-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts() | undefined). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> - safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{}). + lookup_value(?SUBOPTION, {SubPid, Topic}); +get_subopts(SubId, Topic) when ?is_subid(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + get_subopts(SubPid, Topic); + undefined -> undefined + end. -spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> @@ -331,9 +344,6 @@ set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> topics() -> emqx_router:topics(). -safe_lookup_element(Tab, Key, Def) -> - try ets:lookup_element(Tab, Key, 2) catch error:badarg -> Def end. - %%------------------------------------------------------------------------------ %% Stats fun %%------------------------------------------------------------------------------ @@ -372,10 +382,15 @@ init([Pool, Id]) -> {ok, #{pool => Pool, id => Id}}. handle_call({subscribe, Topic}, _From, State) -> - Ok = case get(Topic) of + Ok = emqx_router:do_add_route(Topic), + {reply, Ok, State}; + +handle_call({subscribe, Topic, I}, _From, State) -> + Ok = case get(Shard = {Topic, I}) of undefined -> - _ = put(Topic, true), - emqx_router:do_add_route(Topic); + _ = put(Shard, true), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {subscribe, Topic}); true -> ok end, {reply, Ok, State}; @@ -384,11 +399,18 @@ handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), {reply, ignored, State}. +handle_cast({subscribe, Topic}, State) -> + case emqx_router:do_add_route(Topic) of + ok -> ok; + {error, Reason} -> + emqx_logger:error("[Broker] Failed to add route: ~p", [Reason]) + end, + {noreply, State}; + handle_cast({unsubscribed, Topic}, State) -> case ets:member(?SUBSCRIBER, Topic) of false -> - _ = erase(Topic), - emqx_router:do_delete_route(Topic); + _ = emqx_router:do_delete_route(Topic); true -> ok end, {noreply, State}; @@ -396,6 +418,7 @@ handle_cast({unsubscribed, Topic}, State) -> handle_cast({unsubscribed, Topic, I}, State) -> case ets:member(?SUBSCRIBER, {shard, Topic, I}) of false -> + _ = erase({Topic, I}), true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), cast(pick(Topic), {unsubscribed, Topic}); true -> ok diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index d3e7f9d37..7d514e31d 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -16,44 +16,56 @@ -behaviour(gen_server). --compile({no_auto_import, [monitor/2]}). - -export([start_link/0]). --export([monitor/2]). --export([get_shard/2]). +-export([register_sub/2]). +-export([lookup_subid/1, lookup_subpid/1]). +-export([get_sub_shard/2]). -export([create_seq/1, reclaim_seq/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(HELPER, ?MODULE). +-define(SUBID, emqx_subid). -define(SUBMON, emqx_submon). -define(SUBSEQ, emqx_subseq). - --record(state, {pmon :: emqx_pmon:pmon()}). +-define(SHARD, 1024). -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). --spec(monitor(pid(), emqx_types:subid()) -> ok). -monitor(SubPid, SubId) when is_pid(SubPid) -> +-spec(register_sub(pid(), emqx_types:subid()) -> ok). +register_sub(SubPid, SubId) when is_pid(SubPid) -> case ets:lookup(?SUBMON, SubPid) of [] -> - gen_server:cast(?HELPER, {monitor, SubPid, SubId}); + gen_server:cast(?HELPER, {register_sub, SubPid, SubId}); [{_, SubId}] -> ok; _Other -> error(subid_conflict) end. --spec(get_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). -get_shard(SubPid, Topic) -> +-spec(lookup_subid(pid()) -> emqx_types:subid() | undefined). +lookup_subid(SubPid) when is_pid(SubPid) -> + emqx_tables:lookup_value(?SUBMON, SubPid). + +-spec(lookup_subpid(emqx_types:subid()) -> pid()). +lookup_subpid(SubId) -> + emqx_tables:lookup_value(?SUBID, SubId). + +-spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_sub_shard(SubPid, Topic) -> case create_seq(Topic) of - Seq when Seq =< 1024 -> 0; - _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shards, 2)) + Seq when Seq =< ?SHARD -> 0; + _ -> erlang:phash2(SubPid, shards_num()) + 1 end. +-spec(shards_num() -> pos_integer()). +shards_num() -> + %% Dynamic sharding later... + ets:lookup_element(?HELPER, shards, 2). + -spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). create_seq(Topic) -> emqx_sequence:nextval(?SUBSEQ, Topic). @@ -67,41 +79,55 @@ reclaim_seq(Topic) -> %%------------------------------------------------------------------------------ init([]) -> + %% Helper table + ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + %% Shards: CPU * 32 + true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), - %% Shards: CPU * 32 - true = ets:insert(?SUBSEQ, {shards, emqx_vm:schedulers() * 32}), + %% SubId: SubId -> SubPid + ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), + ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer - emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), - {ok, #state{pmon = emqx_pmon:new()}, hibernate}. + ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), + {ok, #{pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]), - {reply, ignored, State}. + {reply, ignored, State}. -handle_cast({monitor, SubPid, SubId}, State = #state{pmon = PMon}) -> +handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> + true = (SubId =:= undefined) orelse ets:insert(?SUBID, {SubId, SubPid}), true = ets:insert(?SUBMON, {SubPid, SubId}), - {noreply, State#state{pmon = emqx_pmon:monitor(SubPid, PMon)}}; + {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMon}) -> - true = ets:delete(?SUBMON, SubPid), - ok = emqx_pool:async_submit(fun emqx_broker:subscriber_down/1, [SubPid]), - {noreply, State#state{pmon = emqx_pmon:erase(SubPid, PMon)}}; +handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #{pmon := PMon}) -> + case ets:lookup(?SUBMON, SubPid) of + [{_, SubId}] -> + ok = emqx_pool:async_submit(fun subscriber_down/2, [SubPid, SubId]); + [] -> + emqx_logger:error("[BrokerHelper] unexpected DOWN: ~p, reason: ~p", [SubPid, Reason]) + end, + {noreply, State#{pmon := emqx_pmon:erase(SubPid, PMon)}}; handle_info(Info, State) -> emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> - _ = emqx_sequence:delete(?SUBSEQ), +terminate(_Reason, _State) -> + true = emqx_sequence:delete(?SUBSEQ), emqx_stats:cancel_update(broker_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. +subscriber_down(SubPid, SubId) -> + true = ets:delete(?SUBMON, SubPid), + true = (SubId =:= undefined) orelse ets:delete_object(?SUBID, {SubId, SubPid}), + emqx_broker:subscriber_down(SubPid). + diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index efeaabc74..c32800a24 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -90,7 +90,7 @@ init([]) -> [Node | Acc] end end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), - emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), + ok = emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl index 022531df5..33bb5edda 100644 --- a/src/emqx_sequence.erl +++ b/src/emqx_sequence.erl @@ -51,6 +51,7 @@ reclaim(Name, Key) -> end. %% @doc Delete the sequence. +-spec(delete(name()) -> boolean()). delete(Name) -> case ets:info(Name, name) of Name -> ets:delete(Name); diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index d178a8ae7..637e44b0c 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -206,7 +206,7 @@ init([]) -> ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), - emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), + ok = emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), {ok, #{session_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl index 9b3ebfeae..fdb106a99 100644 --- a/src/emqx_tables.erl +++ b/src/emqx_tables.erl @@ -15,6 +15,7 @@ -module(emqx_tables). -export([new/2]). +-export([lookup_value/2, lookup_value/3]). %% Create a named_table ets. -spec(new(atom(), list()) -> ok). @@ -26,3 +27,16 @@ new(Tab, Opts) -> Tab -> ok end. +%% KV lookup +-spec(lookup_value(atom(), term()) -> any()). +lookup_value(Tab, Key) -> + lookup_value(Tab, Key, undefined). + +-spec(lookup_value(atom(), term(), any()) -> any()). +lookup_value(Tab, Key, Def) -> + try + ets:lookup_element(Tab, Key, 2) + catch + error:badarg -> Def + end. + diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl index f37b60d76..1ac0ea308 100644 --- a/test/emqx_sequence_SUITE.erl +++ b/test/emqx_sequence_SUITE.erl @@ -33,5 +33,6 @@ sequence_generate(_) -> ?assertEqual(1, reclaim(seqtab, key)), ?assertEqual(0, reclaim(seqtab, key)), ?assertEqual(false, ets:member(seqtab, key)), - ?assertEqual(1, nextval(seqtab, key)). + ?assertEqual(1, nextval(seqtab, key)), + ?assert(emqx_sequence:delete(seqtab). diff --git a/test/emqx_tables_SUITE.erl b/test/emqx_tables_SUITE.erl index 95590b0e9..1002c0a0b 100644 --- a/test/emqx_tables_SUITE.erl +++ b/test/emqx_tables_SUITE.erl @@ -20,7 +20,7 @@ all() -> [t_new]. t_new(_) -> - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - ets:insert(TId, {loss, 100}), - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - 100 = ets:lookup_element(TId, loss, 2). + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + ets:insert(test_table, {key, 100}), + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + 100 = ets:lookup_element(test_table, key, 2). From 33830d812006410b0b04d042625e8b8fe5616bd9 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Mon, 10 Dec 2018 18:37:42 +0800 Subject: [PATCH 18/54] Improve the design of trie, router and broker modules 1. Add do_add_route/1 do_add_route/2, do_delete_route/1, do_delete_route/2 APIs in emqx_router module 2. Improve the code of emqx_trie module 3. Update the emqx_broker module to call the new APIs of emqx_router --- src/emqx_broker.erl | 30 ++++----- src/emqx_router.erl | 134 ++++++++++++++++++------------------- src/emqx_router_helper.erl | 45 +++++++------ src/emqx_shared_sub.erl | 7 +- src/emqx_trie.erl | 18 ++--- test/emqx_router_SUITE.erl | 103 ++++++++++------------------ test/emqx_trie_SUITE.erl | 113 ++++++++++++++++--------------- 7 files changed, 208 insertions(+), 242 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index a2ddf5b60..a00dc17b8 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -27,6 +27,7 @@ -export([subscriptions/1, subscribers/1, subscribed/2]). -export([get_subopts/2, set_subopts/2]). -export([topics/0]). + %% Stats fun -export([stats_fun/0]). @@ -52,9 +53,9 @@ -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - _ = create_tabs(), - Name = emqx_misc:proc_name(?BROKER, Id), - gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], []). + ok = create_tabs(), + gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, + ?MODULE, [Pool, Id], []). %%------------------------------------------------------------------------------ %% Create tabs @@ -75,7 +76,7 @@ create_tabs() -> ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... - %% duplicate_bag: o(1) insert + %% bag: o(n) insert ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ @@ -114,7 +115,7 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map call(pick({Topic, Shard}), {subscribe, Topic}); Group -> %% Shard subscription true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - emqx_shard_sub:subscribe(Group, Topic, SubPid) + emqx_shared_sub:subscribe(Group, Topic, SubPid) end; true -> ok end. @@ -367,18 +368,17 @@ pick(Topic) -> %%------------------------------------------------------------------------------ init([Pool, Id]) -> - _ = emqx_router:set_mode(protected), true = gproc_pool:connect_worker(Pool, {Pool, Id}), {ok, #{pool => Pool, id => Id}}. handle_call({subscribe, Topic}, _From, State) -> - case get(Topic) of - undefined -> - _ = put(Topic, true), - emqx_router:add_route(Topic); - true -> ok - end, - {reply, ok, State}; + Ok = case get(Topic) of + undefined -> + _ = put(Topic, true), + emqx_router:do_add_route(Topic); + true -> ok + end, + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), @@ -387,8 +387,8 @@ handle_call(Req, _From, State) -> handle_cast({unsubscribed, Topic}, State) -> case ets:member(?SUBSCRIBER, Topic) of false -> - _ = erase(Topic), - emqx_router:delete_route(Topic); + _ = erase(Topic), + emqx_router:do_delete_route(Topic); true -> ok end, {noreply, State}; diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 313adc475..c165b97ca 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -29,19 +29,19 @@ %% Route APIs -export([add_route/1, add_route/2]). --export([get_routes/1]). +-export([do_add_route/1, do_add_route/2]). +-export([match_routes/1, lookup_routes/1, has_routes/1]). -export([delete_route/1, delete_route/2]). --export([has_routes/1, match_routes/1, print_routes/1]). +-export([do_delete_route/1, do_delete_route/2]). +-export([print_routes/1]). -export([topics/0]). -%% Mode --export([set_mode/1, get_mode/0]). - %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --type(destination() :: node() | {binary(), node()}). +-type(group() :: binary()). +-type(destination() :: node() | {group(), node()}). -define(ROUTE, emqx_route). @@ -66,75 +66,76 @@ mnesia(copy) -> -spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - Name = emqx_misc:proc_name(?MODULE, Id), - gen_server:start_link({local, Name}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). %%------------------------------------------------------------------------------ %% Route APIs %%------------------------------------------------------------------------------ --spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). +-spec(add_route(emqx_topic:topic()) -> ok | {error, term()}). add_route(Topic) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = node()}); -add_route(Route = #route{topic = Topic}) -> - case get_mode() of - protected -> do_add_route(Route); - undefined -> call(pick(Topic), {add_route, Route}) - end. + add_route(Topic, node()). -spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). add_route(Topic, Dest) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = Dest}). + call(pick(Topic), {add_route, Topic, Dest}). -%% @private -do_add_route(Route = #route{topic = Topic, dest = Dest}) -> - case lists:member(Route, get_routes(Topic)) of +-spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}). +do_add_route(Topic) when is_binary(Topic) -> + do_add_route(Topic, node()). + +-spec(do_add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_add_route(Topic, Dest) when is_binary(Topic) -> + Route = #route{topic = Topic, dest = Dest}, + case lists:member(Route, lookup_routes(Topic)) of true -> ok; false -> ok = emqx_router_helper:monitor(Dest), case emqx_topic:wildcard(Topic) of - true -> trans(fun add_trie_route/1, [Route]); - false -> add_direct_route(Route) + true -> trans(fun insert_trie_route/1, [Route]); + false -> insert_direct_route(Route) end end. --spec(get_routes(emqx_topic:topic()) -> [emqx_types:route()]). -get_routes(Topic) -> +%% @doc Match routes +-spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). +match_routes(Topic) when is_binary(Topic) -> + %% Optimize: routing table will be replicated to all router nodes. + Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), + lists:append([lookup_routes(To) || To <- [Topic | Matched]]). + +-spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). +lookup_routes(Topic) -> ets:lookup(?ROUTE, Topic). --spec(delete_route(emqx_topic:topic() | emqx_types:route()) -> ok | {error, term()}). -delete_route(Topic) when is_binary(Topic) -> - delete_route(#route{topic = Topic, dest = node()}); -delete_route(Route = #route{topic = Topic}) -> - case get_mode() of - protected -> do_delete_route(Route); - undefined -> call(pick(Topic), {delete_route, Route}) - end. - --spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). -delete_route(Topic, Dest) when is_binary(Topic) -> - delete_route(#route{topic = Topic, dest = Dest}). - -%% @private -do_delete_route(Route = #route{topic = Topic}) -> - case emqx_topic:wildcard(Topic) of - true -> trans(fun del_trie_route/1, [Route]); - false -> del_direct_route(Route) - end. - -spec(has_routes(emqx_topic:topic()) -> boolean()). has_routes(Topic) when is_binary(Topic) -> ets:member(?ROUTE, Topic). --spec(topics() -> list(emqx_topic:topic())). -topics() -> mnesia:dirty_all_keys(?ROUTE). +-spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}). +delete_route(Topic) when is_binary(Topic) -> + delete_route(Topic, node()). -%% @doc Match routes -%% Optimize: routing table will be replicated to all router nodes. --spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). -match_routes(Topic) when is_binary(Topic) -> - Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), - lists:append([get_routes(To) || To <- [Topic | Matched]]). +-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +delete_route(Topic, Dest) when is_binary(Topic) -> + call(pick(Topic), {delete_route, Topic, Dest}). + +-spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}). +do_delete_route(Topic) when is_binary(Topic) -> + do_delete_route(Topic, node()). + +-spec(do_delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_delete_route(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> trans(fun delete_trie_route/1, [Route]); + false -> delete_direct_route(Route) + end. + +-spec(topics() -> list(emqx_topic:topic())). +topics() -> + mnesia:dirty_all_keys(?ROUTE). %% @doc Print routes to a topic -spec(print_routes(emqx_topic:topic()) -> ok). @@ -143,13 +144,6 @@ print_routes(Topic) -> io:format("~s -> ~s~n", [To, Dest]) end, match_routes(Topic)). --spec(set_mode(protected | atom()) -> any()). -set_mode(Mode) when is_atom(Mode) -> - put('$router_mode', Mode). - --spec(get_mode() -> protected | undefined | atom()). -get_mode() -> get('$router_mode'). - call(Router, Msg) -> gen_server:call(Router, Msg, infinity). @@ -164,11 +158,13 @@ init([Pool, Id]) -> true = gproc_pool:connect_worker(Pool, {Pool, Id}), {ok, #{pool => Pool, id => Id}}. -handle_call({add_route, Route}, _From, State) -> - {reply, do_add_route(Route), State}; +handle_call({add_route, Topic, Dest}, _From, State) -> + Ok = do_add_route(Topic, Dest), + {reply, Ok, State}; -handle_call({delete_route, Route}, _From, State) -> - {reply, do_delete_route(Route), State}; +handle_call({delete_route, Topic, Dest}, _From, State) -> + Ok = do_delete_route(Topic, Dest), + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Router] unexpected call: ~p", [Req]), @@ -192,23 +188,23 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -add_direct_route(Route) -> +insert_direct_route(Route) -> mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]). -add_trie_route(Route = #route{topic = Topic}) -> +insert_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [] -> emqx_trie:insert(Topic); _ -> ok end, mnesia:write(?ROUTE, Route, sticky_write). -del_direct_route(Route) -> +delete_direct_route(Route) -> mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]). -del_trie_route(Route = #route{topic = Topic}) -> +delete_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [Route] -> %% Remove route and trie - mnesia:delete_object(?ROUTE, Route, sticky_write), + ok = mnesia:delete_object(?ROUTE, Route, sticky_write), emqx_trie:delete(Topic); [_|_] -> %% Remove route only mnesia:delete_object(?ROUTE, Route, sticky_write); @@ -219,7 +215,7 @@ del_trie_route(Route = #route{topic = Topic}) -> -spec(trans(function(), list(any())) -> ok | {error, term()}). trans(Fun, Args) -> case mnesia:transaction(Fun, Args) of - {atomic, _} -> ok; - {aborted, Error} -> {error, Error} + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} end. diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index c24b10715..efeaabc74 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -31,15 +31,11 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -%% internal export +%% Internal export -export([stats_fun/0]). -record(routing_node, {name, const = unused}). --record(state, {nodes = []}). --compile({no_auto_import, [monitor/1]}). - --define(SERVER, ?MODULE). -define(ROUTE, emqx_route). -define(ROUTING_NODE, emqx_routing_node). -define(LOCK, {?MODULE, cleanup_routes}). @@ -64,9 +60,9 @@ mnesia(copy) -> %%------------------------------------------------------------------------------ %% @doc Starts the router helper --spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). %% @doc Monitor routing node -spec(monitor(node() | {binary(), node()}) -> ok). @@ -84,18 +80,18 @@ monitor(Node) when is_atom(Node) -> %%------------------------------------------------------------------------------ init([]) -> - _ = ekka:monitor(membership), - _ = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + ok = ekka:monitor(membership), + {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of true -> Acc; - false -> _ = erlang:monitor_node(Node, true), + false -> true = erlang:monitor_node(Node, true), [Node | Acc] end end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), - {ok, #state{nodes = Nodes}, hibernate}. + {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> emqx_logger:error("[RouterHelper] unexpected call: ~p", [Req]), @@ -105,24 +101,29 @@ handle_cast(Msg, State) -> emqx_logger:error("[RouterHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({mnesia_table_event, {write, #routing_node{name = Node}, _}}, State = #state{nodes = Nodes}) -> - emqx_logger:info("[RouterHelper] write routing node: ~s", [Node]), +handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, State = #{nodes := Nodes}) -> case ekka:is_member(Node) orelse lists:member(Node, Nodes) of - true -> {noreply, State}; - false -> _ = erlang:monitor_node(Node, true), - {noreply, State#state{nodes = [Node | Nodes]}} + true -> {noreply, State}; + false -> + true = erlang:monitor_node(Node, true), + {noreply, State#{nodes := [Node | Nodes]}} end; -handle_info({mnesia_table_event, _Event}, State) -> +handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> + %% ignore {noreply, State}; -handle_info({nodedown, Node}, State = #state{nodes = Nodes}) -> +handle_info({mnesia_table_event, Event}, State) -> + emqx_logger:error("[RouterHelper] unexpected mnesia_table_event: ~p", [Event]), + {noreply, State}; + +handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> global:trans({?LOCK, self()}, fun() -> mnesia:transaction(fun cleanup_routes/1, [Node]) end), - mnesia:dirty_delete(?ROUTING_NODE, Node), - {noreply, State#state{nodes = lists:delete(Node, Nodes)}, hibernate}; + ok = mnesia:dirty_delete(?ROUTING_NODE, Node), + {noreply, State#{nodes := lists:delete(Node, Nodes)}, hibernate}; handle_info({membership, {mnesia, down, Node}}, State) -> handle_info({nodedown, Node}, State); @@ -134,8 +135,8 @@ handle_info(Info, State) -> emqx_logger:error("[RouteHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> - ekka:unmonitor(membership), +terminate(_Reason, _State) -> + ok = ekka:unmonitor(membership), emqx_stats:cancel_update(route_stats), mnesia:unsubscribe({table, ?ROUTING_NODE, simple}). diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index d1d0d921d..b7e41213b 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -252,7 +252,6 @@ subscribers(Group, Topic) -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_router:set_mode(protected), mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mnesia:transaction(fun init_monitors/0), ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), @@ -269,7 +268,7 @@ handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:add_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) end, ok = maybe_insert_alive_tab(SubPid), true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), @@ -280,7 +279,7 @@ handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end, {reply, ok, State}; @@ -334,7 +333,7 @@ cleanup_down(SubPid) -> true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), case ets:member(?SHARED_SUBS, {Group, Topic}) of true -> ok; - false -> ok = emqx_router:delete_route(Topic, {Group, node()}) + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 79f6042b7..27ff52827 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -36,7 +36,7 @@ %% @doc Create or replicate trie tables. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> - %% Optimize + %% Optimize storage StoreProps = [{ets, [{read_concurrency, true}, {write_concurrency, true}]}], %% Trie table @@ -72,7 +72,7 @@ insert(Topic) when is_binary(Topic) -> write_trie_node(TrieNode#trie_node{topic = Topic}); [] -> %% Add trie path - lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), + ok = lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), %% Add last node write_trie_node(#trie_node{node_id = Topic, topic = Topic}) end. @@ -93,7 +93,7 @@ lookup(NodeId) -> delete(Topic) when is_binary(Topic) -> case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{edge_count = 0}] -> - mnesia:delete({?TRIE_NODE, Topic}), + ok = mnesia:delete({?TRIE_NODE, Topic}), delete_path(lists:reverse(emqx_topic:triples(Topic))); [TrieNode] -> write_trie_node(TrieNode#trie_node{topic = undefined}); @@ -112,12 +112,12 @@ add_path({Node, Word, Child}) -> [TrieNode = #trie_node{edge_count = Count}] -> case mnesia:wread({?TRIE, Edge}) of [] -> - write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), + ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), write_trie(#trie{edge = Edge, node_id = Child}); [_] -> ok end; [] -> - write_trie_node(#trie_node{node_id = Node, edge_count = 1}), + ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}), write_trie(#trie{edge = Edge, node_id = Child}) end. @@ -154,10 +154,10 @@ match_node(NodeId, [W|Words], ResAcc) -> delete_path([]) -> ok; delete_path([{NodeId, Word, _} | RestPath]) -> - mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), - case mnesia:read(?TRIE_NODE, NodeId) of + ok = mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), + case mnesia:wread({?TRIE_NODE, NodeId}) of [#trie_node{edge_count = 1, topic = undefined}] -> - mnesia:delete({?TRIE_NODE, NodeId}), + ok = mnesia:delete({?TRIE_NODE, NodeId}), delete_path(RestPath); [TrieNode = #trie_node{edge_count = 1, topic = _}] -> write_trie_node(TrieNode#trie_node{edge_count = 0}); @@ -167,9 +167,11 @@ delete_path([{NodeId, Word, _} | RestPath]) -> mnesia:abort({node_not_found, NodeId}) end. +%% @private write_trie(Trie) -> mnesia:write(?TRIE, Trie, write). +%% @private write_trie_node(TrieNode) -> mnesia:write(?TRIE_NODE, TrieNode, write). diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl index e317ec7b3..c115fd0cd 100644 --- a/test/emqx_router_SUITE.erl +++ b/test/emqx_router_SUITE.erl @@ -21,17 +21,16 @@ -compile(nowarn_export_all). -define(R, emqx_router). --define(TABS, [emqx_route, emqx_trie, emqx_trie_node]). all() -> [{group, route}]. groups() -> [{route, [sequence], - [add_del_route, - match_routes, - has_routes, - router_add_del]}]. + [t_add_delete, + t_do_add_delete, + t_match_routes, + t_has_routes]}]. init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), @@ -47,77 +46,47 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). -add_del_route(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/+/b">>, node()), - ct:log("Topics: ~p ~n", [emqx_topic:wildcard(<<"a/+/b">>)]), - timer:sleep(1), - +t_add_delete(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/b">>, node()), ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), - ?R:del_route(From, <<"a/b/c">>, node()), + ?R:delete_route(<<"a/b/c">>), + ?R:delete_route(<<"a/+/b">>, node()), + ?assertEqual([], ?R:topics()). - ?R:del_route(From, <<"a/+/b">>, node()), - timer:sleep(120), - ?assertEqual([], lists:sort(?R:topics())). +t_do_add_delete(_) -> + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/+/b">>, node()), + ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), -match_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - ?R:add_route(From, <<"a/+/c">>, node()), - ?R:add_route(From, <<"a/b/#">>, node()), - ?R:add_route(From, <<"#">>, node()), - timer:sleep(1000), + ?R:do_delete_route(<<"a/b/c">>, node()), + ?R:do_delete_route(<<"a/+/b">>), + ?assertEqual([], ?R:topics()). + +t_match_routes(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/c">>, node()), + ?R:add_route(<<"a/b/#">>, node()), + ?R:add_route(<<"#">>, node()), ?assertEqual([#route{topic = <<"#">>, dest = node()}, #route{topic = <<"a/+/c">>, dest = node()}, #route{topic = <<"a/b/#">>, dest = node()}, #route{topic = <<"a/b/c">>, dest = node()}], - lists:sort(?R:match_routes(<<"a/b/c">>))). + lists:sort(?R:match_routes(<<"a/b/c">>))), + ?R:delete_route(<<"a/b/c">>, node()), + ?R:delete_route(<<"a/+/c">>, node()), + ?R:delete_route(<<"a/b/#">>, node()), + ?R:delete_route(<<"#">>, node()), + ?assertEqual([], lists:sort(?R:match_routes(<<"a/b/c">>))). -has_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"devices/+/messages">>, node()), - timer:sleep(200), - ?assert(?R:has_routes(<<"devices/+/messages">>)). +t_has_routes(_) -> + ?R:add_route(<<"devices/+/messages">>, node()), + ?assert(?R:has_routes(<<"devices/+/messages">>)), + ?R:delete_route(<<"devices/+/messages">>). clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, ?TABS). - -router_add_del(_) -> - ?R:add_route(<<"#">>), - ?R:add_route(<<"a/b/c">>, node()), - ?R:add_route(<<"+/#">>), - Routes = [R1, R2 | _] = [ - #route{topic = <<"#">>, dest = node()}, - #route{topic = <<"+/#">>, dest = node()}, - #route{topic = <<"a/b/c">>, dest = node()}], - timer:sleep(500), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - ?R:print_routes(<<"a/b/c">>), - - %% Batch Add - lists:foreach(fun(R) -> ?R:add_route(R) end, Routes), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - %% Del - ?R:del_route(<<"a/b/c">>, node()), - timer:sleep(500), - [R1, R2] = lists:sort(?R:match_routes(<<"a/b/c">>)), - {atomic, []} = mnesia:transaction(fun emqx_trie:lookup/1, [<<"a/b/c">>]), - - %% Batch Del - R3 = #route{topic = <<"#">>, dest = 'a@127.0.0.1'}, - ?R:add_route(R3), - ?R:del_route(<<"#">>), - ?R:del_route(R2), - ?R:del_route(R3), - timer:sleep(500), - [] = lists:sort(?R:match_routes(<<"a/b/c">>)). + lists:foreach(fun mnesia:clear_table/1, [emqx_route, emqx_trie, emqx_trie_node]). diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 85637a447..09226979f 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -47,36 +47,36 @@ t_insert(_) -> edge_count = 3, topic = <<"sensor">>, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:lookup(<<"sensor">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:lookup(<<"sensor">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_match(_) -> Machted = [<<"sensor/+/#">>, <<"sensor/#">>], - {atomic, Machted} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:match(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:match(<<"sensor/1">>) + end, + ?assertEqual({atomic, Machted}, mnesia:transaction(Fun)). t_match2(_) -> Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, - {atomic, Matched} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"#">>), - ?TRIE:insert(<<"+/#">>), - ?TRIE:insert(<<"+/+/#">>), - {?TRIE:match(<<"a/b/c">>), - ?TRIE:match(<<"$SYS/broker/zenmq">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"#">>), + ?TRIE:insert(<<"+/#">>), + ?TRIE:insert(<<"+/+/#">>), + {?TRIE:match(<<"a/b/c">>), + ?TRIE:match(<<"$SYS/broker/zenmq">>)} + end, + ?assertEqual({atomic, Matched}, mnesia:transaction(Fun)). t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], @@ -91,43 +91,42 @@ t_delete(_) -> edge_count = 2, topic = undefined, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/#">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:lookup(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/#">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:lookup(<<"sensor/1">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_delete2(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric/3">>), - {?TRIE:lookup(<<"sensor">>), - ?TRIE:lookup(<<"sensor/1">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric/3">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). t_delete3(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/+">>), - ?TRIE:insert(<<"sensor/+/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor/+/metric/2">>), - ?TRIE:delete(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/+">>), - ?TRIE:delete(<<"sensor/+/unknown">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/+">>), + ?TRIE:insert(<<"sensor/+/metric/2">>), + ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor/+/metric/2">>), + ?TRIE:delete(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/+">>), + ?TRIE:delete(<<"sensor/+/unknown">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). clear_tables() -> lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS). From b279eff18106b53b1c75635bd26a0717a4807fd2 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 11 Dec 2018 14:06:23 +0800 Subject: [PATCH 19/54] Add t_mnesia/1 test case --- test/emqx_trie_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 09226979f..500fe3574 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -24,7 +24,7 @@ -define(TRIE_TABS, [emqx_trie, emqx_trie_node]). all() -> - [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. + [t_mnesia, t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. init_per_suite(Config) -> application:load(emqx), @@ -42,6 +42,9 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). +t_mnesia(_) -> + ok = ?TRIE:mnesia(copy). + t_insert(_) -> TN = #trie_node{node_id = <<"sensor">>, edge_count = 3, From 2a747c9d538cd682c6228a6a799470089b9ed90d Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 13:34:13 +0800 Subject: [PATCH 20/54] Improve the subscription sharding. --- src/emqx.erl | 6 +- src/emqx_broker.erl | 217 +++++++++++++++++++---------------- src/emqx_broker_helper.erl | 80 ++++++++----- src/emqx_router_helper.erl | 2 +- src/emqx_sequence.erl | 1 + src/emqx_sm.erl | 2 +- src/emqx_tables.erl | 14 +++ test/emqx_sequence_SUITE.erl | 3 +- test/emqx_tables_SUITE.erl | 8 +- 9 files changed, 197 insertions(+), 136 deletions(-) diff --git a/src/emqx.erl b/src/emqx.erl index 3792cc4f8..76e966a59 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -22,7 +22,7 @@ %% PubSub API -export([subscribe/1, subscribe/2, subscribe/3]). -export([publish/1]). --export([unsubscribe/1, unsubscribe/2]). +-export([unsubscribe/1]). %% PubSub management API -export([topics/0, subscriptions/1, subscribers/1, subscribed/2]). @@ -88,10 +88,6 @@ publish(Msg) -> unsubscribe(Topic) -> emqx_broker:unsubscribe(iolist_to_binary(Topic)). --spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid()) -> ok). -unsubscribe(Topic, SubId) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId). - %%------------------------------------------------------------------------------ %% PubSub management API %%------------------------------------------------------------------------------ diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index a00dc17b8..9ed4aad06 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -20,7 +20,7 @@ -export([start_link/2]). -export([subscribe/1, subscribe/2, subscribe/3]). --export([unsubscribe/1, unsubscribe/2]). +-export([unsubscribe/1]). -export([subscriber_down/1]). -export([publish/1, safe_publish/1]). -export([dispatch/2]). @@ -35,6 +35,8 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-import(emqx_tables, [lookup_value/2, lookup_value/3]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -42,8 +44,7 @@ -define(BROKER, ?MODULE). -%% ETS tables --define(SUBID, emqx_subid). +%% ETS tables for PubSub -define(SUBOPTION, emqx_suboption). -define(SUBSCRIBER, emqx_subscriber). -define(SUBSCRIPTION, emqx_subscription). @@ -65,9 +66,6 @@ start_link(Pool, Id) -> create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], - %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [set | TabOpts]), - %% SubOption: {SubPid, Topic} -> SubOption ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]), @@ -76,7 +74,7 @@ create_tabs() -> ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... - %% bag: o(n) insert + %% bag: o(n) insert:( ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ @@ -98,28 +96,37 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map SubPid = self(), case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> - ok = emqx_broker_helper:monitor(SubPid, SubId), - %% true = ets:insert(?SUBID, {SubId, SubPid}), - true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), - case maps:get(share, SubOpts, undefined) of - undefined -> - Shard = emqx_broker_helper:get_shard(SubPid, Topic), - case Shard of - 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}); - I -> - true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}) - end, - SubOpts1 = maps:put(shard, Shard, SubOpts), - true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts1}), - call(pick({Topic, Shard}), {subscribe, Topic}); - Group -> %% Shard subscription - true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), - emqx_shared_sub:subscribe(Group, Topic, SubPid) - end; + ok = emqx_broker_helper:monitor_sub(SubPid, SubId), + do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); true -> ok end. +with_subid(undefined, SubOpts) -> + SubOpts; +with_subid(SubId, SubOpts) -> + maps:put(subid, SubId, SubOpts). + +%% @private +do_subscribe(Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_subscribe(Group, Topic, SubPid, SubOpts). + +do_subscribe(undefined, Topic, SubPid, SubOpts) -> + case emqx_broker_helper:get_sub_shard(SubPid, Topic) of + 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + call(pick(Topic), {subscribe, Topic}); + I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, maps:put(shard, I, SubOpts)}), + call(pick({Topic, I}), {subscribe, Topic, I}) + end; + +%% Shared subscription +do_subscribe(Group, Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + emqx_shared_sub:subscribe(Group, Topic, SubPid). + %%------------------------------------------------------------------------------ %% Unsubscribe API %%------------------------------------------------------------------------------ @@ -130,33 +137,26 @@ unsubscribe(Topic) when is_binary(Topic) -> case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> _ = emqx_broker_helper:reclaim_seq(Topic), - case maps:get(share, SubOpts, undefined) of - undefined -> - case maps:get(shard, SubOpts, 0) of - 0 -> - true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> - true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shard, Topic, I}) of - true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) - end, - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; - Group -> - ok = emqx_shared_sub:unsubscribe(Group, Topic, SubPid) - end, - true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), - %%true = ets:delete_object(?SUBID, {SubId, SubPid}), - true = ets:delete(?SUBOPTION, {SubPid, Topic}), - ok; + do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok end. --spec(unsubscribe(emqx_topic:topic(), emqx_types:subid()) -> ok). -unsubscribe(Topic, _SubId) when is_binary(Topic) -> - unsubscribe(Topic). +do_unsubscribe(Topic, SubPid, SubOpts) -> + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_unsubscribe(Group, Topic, SubPid, SubOpts). + +do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + +do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> + emqx_shared_sub:unsubscribe(Group, Topic, SubPid). %%------------------------------------------------------------------------------ %% Publish @@ -241,23 +241,28 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> inc_dropped_cnt(Topic), Delivery; [Sub] -> %% optimize? - dispatch(Sub, Topic, Msg), - Delivery#delivery{results = [{dispatch, Topic, 1}|Results]}; + Cnt = dispatch(Sub, Topic, Msg), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]}; Subs -> - Count = lists:foldl( - fun(Sub, Acc) -> - dispatch(Sub, Topic, Msg), Acc + 1 - end, 0, Subs), - Delivery#delivery{results = [{dispatch, Topic, Count}|Results]} + Cnt = lists:foldl( + fun(Sub, Acc) -> + dispatch(Sub, Topic, Msg) + Acc + end, 0, Subs), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]} end. dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; + case erlang:is_process_alive(SubPid) of + true -> + SubPid ! {dispatch, Topic, Msg}, + 1; + false -> 0 + end; dispatch({shard, I}, Topic, Msg) -> - - lists:foreach(fun(SubPid) -> - SubPid ! {dispatch, Topic, Msg} - end, safe_lookup_element(?SUBSCRIBER, {shard, Topic, I}, [])). + lists:foldl( + fun(SubPid, Cnt) -> + dispatch(SubPid, Topic, Msg) + Cnt + end, 0, subscribers({shard, Topic, I})). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; @@ -265,8 +270,10 @@ inc_dropped_cnt(_Topic) -> emqx_metrics:inc('messages/dropped'). -spec(subscribers(emqx_topic:topic()) -> [pid()]). -subscribers(Topic) -> - safe_lookup_element(?SUBSCRIBER, Topic, []). +subscribers(Topic) when is_binary(Topic) -> + lookup_value(?SUBSCRIBER, Topic, []); +subscribers(Shard = {shard, _Topic, _I}) -> + lookup_value(?SUBSCRIBER, Shard, []). %%------------------------------------------------------------------------------ %% Subscriber is down @@ -275,27 +282,21 @@ subscribers(Topic) -> -spec(subscriber_down(pid()) -> true). subscriber_down(SubPid) -> lists:foreach( - fun(Sub = {_Pid, Topic}) -> - case ets:lookup(?SUBOPTION, Sub) of - [{_, SubOpts}] -> + fun(Topic) -> + case lookup_value(?SUBOPTION, {SubPid, Topic}) of + SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), + true = ets:delete(?SUBOPTION, {SubPid, Topic}), case maps:get(shard, SubOpts, 0) of - 0 -> - true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> - true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - case ets:member(emqx_subscriber, {shard, Topic, I}) of - true -> ok; - false -> ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}) - end, - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end, - ets:delete(?SUBOPTION, Sub); - [] -> ok + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + undefined -> ok end - end, ets:lookup(?SUBSCRIPTION, SubPid)), - true = ets:delete(?SUBSCRIPTION, SubPid). + end, lookup_value(?SUBSCRIPTION, SubPid, [])), + ets:delete(?SUBSCRIPTION, SubPid). %%------------------------------------------------------------------------------ %% Management APIs @@ -303,20 +304,32 @@ subscriber_down(SubPid) -> -spec(subscriptions(pid() | emqx_types:subid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(SubPid) -> - [{Topic, safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{})} - || Topic <- safe_lookup_element(?SUBSCRIPTION, SubPid, [])]. +subscriptions(SubPid) when is_pid(SubPid) -> + [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})} + || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])]; +subscriptions(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + subscriptions(SubPid); + undefined -> [] + end. -spec(subscribed(pid(), emqx_topic:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); subscribed(SubId, Topic) when ?is_subid(SubId) -> - %%FIXME:... SubId -> SubPid - ets:member(?SUBOPTION, {SubId, Topic}). + SubPid = emqx_broker_helper:lookup_subpid(SubId), + ets:member(?SUBOPTION, {SubPid, Topic}). --spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts()). +-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts() | undefined). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> - safe_lookup_element(?SUBOPTION, {SubPid, Topic}, #{}). + lookup_value(?SUBOPTION, {SubPid, Topic}); +get_subopts(SubId, Topic) when ?is_subid(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + get_subopts(SubPid, Topic); + undefined -> undefined + end. -spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> @@ -331,9 +344,6 @@ set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> topics() -> emqx_router:topics(). -safe_lookup_element(Tab, Key, Def) -> - try ets:lookup_element(Tab, Key, 2) catch error:badarg -> Def end. - %%------------------------------------------------------------------------------ %% Stats fun %%------------------------------------------------------------------------------ @@ -372,10 +382,15 @@ init([Pool, Id]) -> {ok, #{pool => Pool, id => Id}}. handle_call({subscribe, Topic}, _From, State) -> - Ok = case get(Topic) of + Ok = emqx_router:do_add_route(Topic), + {reply, Ok, State}; + +handle_call({subscribe, Topic, I}, _From, State) -> + Ok = case get(Shard = {Topic, I}) of undefined -> - _ = put(Topic, true), - emqx_router:do_add_route(Topic); + _ = put(Shard, true), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {subscribe, Topic}); true -> ok end, {reply, Ok, State}; @@ -384,11 +399,18 @@ handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), {reply, ignored, State}. +handle_cast({subscribe, Topic}, State) -> + case emqx_router:do_add_route(Topic) of + ok -> ok; + {error, Reason} -> + emqx_logger:error("[Broker] Failed to add route: ~p", [Reason]) + end, + {noreply, State}; + handle_cast({unsubscribed, Topic}, State) -> case ets:member(?SUBSCRIBER, Topic) of false -> - _ = erase(Topic), - emqx_router:do_delete_route(Topic); + _ = emqx_router:do_delete_route(Topic); true -> ok end, {noreply, State}; @@ -396,6 +418,7 @@ handle_cast({unsubscribed, Topic}, State) -> handle_cast({unsubscribed, Topic, I}, State) -> case ets:member(?SUBSCRIBER, {shard, Topic, I}) of false -> + _ = erase({Topic, I}), true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), cast(pick(Topic), {unsubscribed, Topic}); true -> ok diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index d3e7f9d37..7d514e31d 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -16,44 +16,56 @@ -behaviour(gen_server). --compile({no_auto_import, [monitor/2]}). - -export([start_link/0]). --export([monitor/2]). --export([get_shard/2]). +-export([register_sub/2]). +-export([lookup_subid/1, lookup_subpid/1]). +-export([get_sub_shard/2]). -export([create_seq/1, reclaim_seq/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(HELPER, ?MODULE). +-define(SUBID, emqx_subid). -define(SUBMON, emqx_submon). -define(SUBSEQ, emqx_subseq). - --record(state, {pmon :: emqx_pmon:pmon()}). +-define(SHARD, 1024). -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). --spec(monitor(pid(), emqx_types:subid()) -> ok). -monitor(SubPid, SubId) when is_pid(SubPid) -> +-spec(register_sub(pid(), emqx_types:subid()) -> ok). +register_sub(SubPid, SubId) when is_pid(SubPid) -> case ets:lookup(?SUBMON, SubPid) of [] -> - gen_server:cast(?HELPER, {monitor, SubPid, SubId}); + gen_server:cast(?HELPER, {register_sub, SubPid, SubId}); [{_, SubId}] -> ok; _Other -> error(subid_conflict) end. --spec(get_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). -get_shard(SubPid, Topic) -> +-spec(lookup_subid(pid()) -> emqx_types:subid() | undefined). +lookup_subid(SubPid) when is_pid(SubPid) -> + emqx_tables:lookup_value(?SUBMON, SubPid). + +-spec(lookup_subpid(emqx_types:subid()) -> pid()). +lookup_subpid(SubId) -> + emqx_tables:lookup_value(?SUBID, SubId). + +-spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_sub_shard(SubPid, Topic) -> case create_seq(Topic) of - Seq when Seq =< 1024 -> 0; - _Seq -> erlang:phash2(SubPid, ets:lookup_element(?SUBSEQ, shards, 2)) + Seq when Seq =< ?SHARD -> 0; + _ -> erlang:phash2(SubPid, shards_num()) + 1 end. +-spec(shards_num() -> pos_integer()). +shards_num() -> + %% Dynamic sharding later... + ets:lookup_element(?HELPER, shards, 2). + -spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). create_seq(Topic) -> emqx_sequence:nextval(?SUBSEQ, Topic). @@ -67,41 +79,55 @@ reclaim_seq(Topic) -> %%------------------------------------------------------------------------------ init([]) -> + %% Helper table + ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + %% Shards: CPU * 32 + true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), - %% Shards: CPU * 32 - true = ets:insert(?SUBSEQ, {shards, emqx_vm:schedulers() * 32}), + %% SubId: SubId -> SubPid + ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [set, protected, {read_concurrency, true}]), + ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer - emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), - {ok, #state{pmon = emqx_pmon:new()}, hibernate}. + ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), + {ok, #{pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]), - {reply, ignored, State}. + {reply, ignored, State}. -handle_cast({monitor, SubPid, SubId}, State = #state{pmon = PMon}) -> +handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> + true = (SubId =:= undefined) orelse ets:insert(?SUBID, {SubId, SubPid}), true = ets:insert(?SUBMON, {SubPid, SubId}), - {noreply, State#state{pmon = emqx_pmon:monitor(SubPid, PMon)}}; + {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMon}) -> - true = ets:delete(?SUBMON, SubPid), - ok = emqx_pool:async_submit(fun emqx_broker:subscriber_down/1, [SubPid]), - {noreply, State#state{pmon = emqx_pmon:erase(SubPid, PMon)}}; +handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #{pmon := PMon}) -> + case ets:lookup(?SUBMON, SubPid) of + [{_, SubId}] -> + ok = emqx_pool:async_submit(fun subscriber_down/2, [SubPid, SubId]); + [] -> + emqx_logger:error("[BrokerHelper] unexpected DOWN: ~p, reason: ~p", [SubPid, Reason]) + end, + {noreply, State#{pmon := emqx_pmon:erase(SubPid, PMon)}}; handle_info(Info, State) -> emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> - _ = emqx_sequence:delete(?SUBSEQ), +terminate(_Reason, _State) -> + true = emqx_sequence:delete(?SUBSEQ), emqx_stats:cancel_update(broker_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. +subscriber_down(SubPid, SubId) -> + true = ets:delete(?SUBMON, SubPid), + true = (SubId =:= undefined) orelse ets:delete_object(?SUBID, {SubId, SubPid}), + emqx_broker:subscriber_down(SubPid). + diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index efeaabc74..c32800a24 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -90,7 +90,7 @@ init([]) -> [Node | Acc] end end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), - emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), + ok = emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl index 022531df5..33bb5edda 100644 --- a/src/emqx_sequence.erl +++ b/src/emqx_sequence.erl @@ -51,6 +51,7 @@ reclaim(Name, Key) -> end. %% @doc Delete the sequence. +-spec(delete(name()) -> boolean()). delete(Name) -> case ets:info(Name, name) of Name -> ets:delete(Name); diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index d178a8ae7..637e44b0c 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -206,7 +206,7 @@ init([]) -> ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), - emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), + ok = emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), {ok, #{session_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl index 9b3ebfeae..fdb106a99 100644 --- a/src/emqx_tables.erl +++ b/src/emqx_tables.erl @@ -15,6 +15,7 @@ -module(emqx_tables). -export([new/2]). +-export([lookup_value/2, lookup_value/3]). %% Create a named_table ets. -spec(new(atom(), list()) -> ok). @@ -26,3 +27,16 @@ new(Tab, Opts) -> Tab -> ok end. +%% KV lookup +-spec(lookup_value(atom(), term()) -> any()). +lookup_value(Tab, Key) -> + lookup_value(Tab, Key, undefined). + +-spec(lookup_value(atom(), term(), any()) -> any()). +lookup_value(Tab, Key, Def) -> + try + ets:lookup_element(Tab, Key, 2) + catch + error:badarg -> Def + end. + diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl index f37b60d76..1ac0ea308 100644 --- a/test/emqx_sequence_SUITE.erl +++ b/test/emqx_sequence_SUITE.erl @@ -33,5 +33,6 @@ sequence_generate(_) -> ?assertEqual(1, reclaim(seqtab, key)), ?assertEqual(0, reclaim(seqtab, key)), ?assertEqual(false, ets:member(seqtab, key)), - ?assertEqual(1, nextval(seqtab, key)). + ?assertEqual(1, nextval(seqtab, key)), + ?assert(emqx_sequence:delete(seqtab). diff --git a/test/emqx_tables_SUITE.erl b/test/emqx_tables_SUITE.erl index 95590b0e9..1002c0a0b 100644 --- a/test/emqx_tables_SUITE.erl +++ b/test/emqx_tables_SUITE.erl @@ -20,7 +20,7 @@ all() -> [t_new]. t_new(_) -> - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - ets:insert(TId, {loss, 100}), - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - 100 = ets:lookup_element(TId, loss, 2). + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + ets:insert(test_table, {key, 100}), + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + 100 = ets:lookup_element(test_table, key, 2). From 99872b253fd2a64c60902143c166890706f184d0 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 14:53:22 +0800 Subject: [PATCH 21/54] Fix 'function not exported' crash --- src/emqx_broker.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 9ed4aad06..105589c65 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -96,7 +96,7 @@ subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map SubPid = self(), case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> - ok = emqx_broker_helper:monitor_sub(SubPid, SubId), + ok = emqx_broker_helper:register_sub(SubPid, SubId), do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); true -> ok end. From 8d50c62a94f0bfdf2aefef97a468614824f40dca Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 16:10:16 +0800 Subject: [PATCH 22/54] Optimize connection and session management --- src/emqx_cm.erl | 24 +++++++---- src/emqx_protocol.erl | 13 +++--- src/emqx_session.erl | 15 ++++--- src/emqx_session_sup.erl | 2 +- src/emqx_sm.erl | 88 ++++++++++++++++++++-------------------- src/emqx_sm_locker.erl | 2 +- src/emqx_sm_registry.erl | 5 +-- 7 files changed, 78 insertions(+), 71 deletions(-) diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 6756cf02b..0d2ecf5eb 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -36,7 +36,7 @@ -define(CM, ?MODULE). %% ETS Tables. --define(CONN_TAB, emqx_conn). +-define(CONN_TAB, emqx_conn). -define(CONN_ATTRS_TAB, emqx_conn_attrs). -define(CONN_STATS_TAB, emqx_conn_stats). @@ -56,7 +56,7 @@ register_connection(ClientId) when is_binary(ClientId) -> register_connection({ClientId, self()}); register_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:insert(?CONN_TAB, Conn), + true = ets:insert(?CONN_TAB, Conn), notify({registered, ClientId, ConnPid}). -spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}, list()) -> ok). @@ -87,10 +87,13 @@ unregister_connection(ClientId) when is_binary(ClientId) -> unregister_connection({ClientId, self()}); unregister_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:delete(?CONN_STATS_TAB, Conn), - _ = ets:delete(?CONN_ATTRS_TAB, Conn), - _ = ets:delete_object(?CONN_TAB, Conn), - notify({unregistered, ClientId, ConnPid}). + do_unregister_connection(Conn), + notify({unregistered, ConnPid}). + +do_unregister_connection(Conn) -> + true = ets:delete(?CONN_STATS_TAB, Conn), + true = ets:delete(?CONN_ATTRS_TAB, Conn), + true = ets:delete_object(?CONN_TAB, Conn). %% @doc Lookup connection pid -spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). @@ -138,7 +141,7 @@ handle_call(Req, _From, State) -> handle_cast({notify, {registered, ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:monitor(ConnPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> +handle_cast({notify, {unregistered, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:demonitor(ConnPid, PMon)}}; handle_cast(Msg, State) -> @@ -150,7 +153,12 @@ handle_info({'DOWN', _MRef, process, ConnPid, _Reason}, State = #{conn_pmon := P undefined -> {noreply, State}; ClientId -> - unregister_connection({ClientId, ConnPid}), + Conn = {ClientId, ConnPid}, + case ets:member(?CONN_ATTRS_TAB, Conn) of + true -> + ok = emqx_pool:async_submit(fun do_unregister_connection/1, [Conn]); + false -> ok + end, {noreply, State#{conn_pmon := emqx_pmon:erase(ConnPid, PMon)}} end; diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 573b913f7..3052191a1 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -849,14 +849,13 @@ shutdown(_Reason, #pstate{client_id = undefined}) -> ok; shutdown(_Reason, #pstate{connected = false}) -> ok; -shutdown(Reason, #pstate{client_id = ClientId}) when Reason =:= conflict; - Reason =:= discard -> - emqx_cm:unregister_connection(ClientId); -shutdown(Reason, PState = #pstate{connected = true, - client_id = ClientId}) -> +shutdown(conflict, _PState) -> + ok; +shutdown(discard, _PState) -> + ok; +shutdown(Reason, PState) -> ?LOG(info, "Shutdown for ~p", [Reason]), - emqx_hooks:run('client.disconnected', [credentials(PState), Reason]), - emqx_cm:unregister_connection(ClientId). + emqx_hooks:run('client.disconnected', [credentials(PState), Reason]). start_keepalive(0, _PState) -> ignore; diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 262a9a7a8..b92baa567 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -645,16 +645,14 @@ handle_info(Info, State) -> emqx_logger:error("[Session] unexpected info: ~p", [Info]), {noreply, State}. -terminate(Reason, #state{will_msg = WillMsg, client_id = ClientId, conn_pid = ConnPid}) -> - emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), +terminate(Reason, #state{will_msg = WillMsg, conn_pid = ConnPid}) -> + %% Should not run hooks here. + %% emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), send_willmsg(WillMsg), %% Ensure to shutdown the connection - if - ConnPid =/= undefined -> - ConnPid ! {shutdown, Reason}; - true -> ok - end, - emqx_sm:unregister_session(ClientId). + (ConnPid =:= undefined) orelse ConnPid ! {shutdown, Reason}. + %% Let it crash. + %% emqx_sm:unregister_session(ClientId). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -1011,3 +1009,4 @@ noreply(State) -> shutdown(Reason, State) -> {stop, {shutdown, Reason}, State}. + diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl index 644e33f37..8efc4afc8 100644 --- a/src/emqx_session_sup.erl +++ b/src/emqx_session_sup.erl @@ -38,7 +38,7 @@ init([]) -> [#{id => session, start => {emqx_session, start_link, []}, restart => temporary, - shutdown => 5000, + shutdown => brutal_kill, type => worker, modules => [emqx_session]}]}}. diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 637e44b0c..df2e4b862 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -40,9 +40,9 @@ -define(SM, ?MODULE). -%% ETS Tables --define(SESSION_TAB, emqx_session). --define(SESSION_P_TAB, emqx_persistent_session). +%% ETS Tables for session management. +-define(SESSION_TAB, emqx_session). +-define(SESSION_P_TAB, emqx_session_p). -define(SESSION_ATTRS_TAB, emqx_session_attrs). -define(SESSION_STATS_TAB, emqx_session_stats). @@ -59,8 +59,7 @@ open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid end, emqx_sm_locker:trans(ClientId, CleanStart); -open_session(SessAttrs = #{clean_start := false, - client_id := ClientId}) -> +open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> ResumeStart = fun(_) -> case resume_session(ClientId, SessAttrs) of {ok, SPid} -> @@ -77,13 +76,14 @@ discard_session(ClientId) when is_binary(ClientId) -> discard_session(ClientId, self()). discard_session(ClientId, ConnPid) when is_binary(ClientId) -> - lists:foreach(fun({_ClientId, SPid}) -> - case catch emqx_session:discard(SPid, ConnPid) of - {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> - emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); - ok -> ok - end - end, lookup_session(ClientId)). + lists:foreach( + fun({_ClientId, SPid}) -> + case catch emqx_session:discard(SPid, ConnPid) of + {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> + emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); + ok -> ok + end + end, lookup_session(ClientId)). %% @doc Try to resume a session. -spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}). @@ -116,19 +116,18 @@ close_session(SPid) when is_pid(SPid) -> register_session(ClientId, SessAttrs) when is_binary(ClientId) -> register_session({ClientId, self()}, SessAttrs); -register_session(Session = {ClientId, SPid}, SessAttrs) - when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_TAB, Session), - ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), - proplists:get_value(clean_start, SessAttrs, true) - andalso ets:insert(?SESSION_P_TAB, Session), - emqx_sm_registry:register_session(Session), +register_session(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) -> + true = ets:insert(?SESSION_TAB, Session), + true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), + true = proplists:get_value(clean_start, SessAttrs, true) + orelse ets:insert(?SESSION_P_TAB, Session), + ok = emqx_sm_registry:register_session(Session), notify({registered, ClientId, SPid}). %% @doc Get session attrs -spec(get_session_attrs({emqx_types:client_id(), pid()}) -> list(emqx_session:attr())). get_session_attrs(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_ATTRS_TAB, Session, []). + emqx_tables:lookup_value(?SESSION_ATTRS_TAB, Session, []). %% @doc Set session attrs -spec(set_session_attrs(emqx_types:client_id() | {emqx_types:client_id(), pid()}, @@ -144,17 +143,21 @@ unregister_session(ClientId) when is_binary(ClientId) -> unregister_session({ClientId, self()}); unregister_session(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - emqx_sm_registry:unregister_session(Session), - ets:delete(?SESSION_STATS_TAB, Session), - ets:delete(?SESSION_ATTRS_TAB, Session), - ets:delete_object(?SESSION_P_TAB, Session), - ets:delete_object(?SESSION_TAB, Session), + ok = do_unregister_session(Session), notify({unregistered, ClientId, SPid}). +%% @private +do_unregister_session(Session) -> + true = ets:delete(?SESSION_STATS_TAB, Session), + true = ets:delete(?SESSION_ATTRS_TAB, Session), + true = ets:delete_object(?SESSION_P_TAB, Session), + true = ets:delete_object(?SESSION_TAB, Session), + emqx_sm_registry:unregister_session(Session). + %% @doc Get session stats -spec(get_session_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). get_session_stats(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_STATS_TAB, Session, []). + emqx_tables:lookup_value(?SESSION_STATS_TAB, Session, []). %% @doc Set session stats -spec(set_session_stats(emqx_types:client_id() | {emqx_types:client_id(), pid()}, @@ -168,7 +171,7 @@ set_session_stats(Session = {ClientId, SPid}, Stats) when is_binary(ClientId), i -spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). lookup_session(ClientId) -> case emqx_sm_registry:is_enabled() of - true -> emqx_sm_registry:lookup_session(ClientId); + true -> emqx_sm_registry:lookup_session(ClientId); false -> ets:lookup(?SESSION_TAB, ClientId) end. @@ -185,13 +188,7 @@ dispatch(ClientId, Topic, Msg) -> %% @doc Lookup session pid. -spec(lookup_session_pid(emqx_types:client_id()) -> pid() | undefined). lookup_session_pid(ClientId) -> - safe_lookup_element(?SESSION_TAB, ClientId, undefined). - -safe_lookup_element(Tab, Key, Default) -> - try ets:lookup_element(Tab, Key, 2) - catch - error:badarg -> Default - end. + emqx_tables:lookup_value(?SESSION_TAB, ClientId). notify(Event) -> gen_server:cast(?SM, {notify, Event}). @@ -207,29 +204,34 @@ init([]) -> ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), ok = emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), - {ok, #{session_pmon => emqx_pmon:new()}}. + {ok, #{sess_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[SM] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({notify, {registered, ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; +handle_cast({notify, {registered, ClientId, SPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:demonitor(SPid, PMon)}}; +handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:demonitor(SPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{session_pmon := PMon}) -> +handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{sess_pmon := PMon}) -> case emqx_pmon:find(DownPid, PMon) of undefined -> {noreply, State}; - ClientId -> - unregister_session({ClientId, DownPid}), - {noreply, State#{session_pmon := emqx_pmon:erase(DownPid, PMon)}} + ClientId -> + Session = {ClientId, DownPid}, + case ets:member(?SESSION_ATTRS_TAB, Session) of + true -> + ok = emqx_pool:async_submit(fun do_unregister_session/1, [Session]); + false -> ok + end, + {noreply, State#{sess_pmon := emqx_pmon:erase(DownPid, PMon)}} end; handle_info(Info, State) -> diff --git a/src/emqx_sm_locker.erl b/src/emqx_sm_locker.erl index 29adf3342..409331b88 100644 --- a/src/emqx_sm_locker.erl +++ b/src/emqx_sm_locker.erl @@ -21,7 +21,7 @@ -export([trans/2, trans/3]). -export([lock/1, lock/2, unlock/1]). --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> ekka_locker:start_link(?MODULE). diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl index b503d71c8..3b472e2c8 100644 --- a/src/emqx_sm_registry.erl +++ b/src/emqx_sm_registry.erl @@ -41,8 +41,7 @@ start_link() -> gen_server:start_link({local, ?REGISTRY}, ?MODULE, [], []). -spec(is_enabled() -> boolean()). -is_enabled() -> - ets:info(?TAB, name) =/= undefined. +is_enabled() -> ets:info(?TAB, name) =/= undefined. -spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), session_pid()})). @@ -73,7 +72,7 @@ init([]) -> {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]), ok = ekka_mnesia:copy_table(?TAB), - _ = ekka:monitor(membership), + ok = ekka:monitor(membership), {ok, #{}}. handle_call(Req, _From, State) -> From 8f2f4b6b812f462f761e0e47ea5c4b357e508565 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 16:10:16 +0800 Subject: [PATCH 23/54] Optimize connection and session management --- src/emqx_cm.erl | 24 +++++++---- src/emqx_protocol.erl | 13 +++--- src/emqx_session.erl | 15 ++++--- src/emqx_session_sup.erl | 2 +- src/emqx_sm.erl | 88 ++++++++++++++++++++-------------------- src/emqx_sm_locker.erl | 2 +- src/emqx_sm_registry.erl | 5 +-- 7 files changed, 78 insertions(+), 71 deletions(-) diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 6756cf02b..0d2ecf5eb 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -36,7 +36,7 @@ -define(CM, ?MODULE). %% ETS Tables. --define(CONN_TAB, emqx_conn). +-define(CONN_TAB, emqx_conn). -define(CONN_ATTRS_TAB, emqx_conn_attrs). -define(CONN_STATS_TAB, emqx_conn_stats). @@ -56,7 +56,7 @@ register_connection(ClientId) when is_binary(ClientId) -> register_connection({ClientId, self()}); register_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:insert(?CONN_TAB, Conn), + true = ets:insert(?CONN_TAB, Conn), notify({registered, ClientId, ConnPid}). -spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}, list()) -> ok). @@ -87,10 +87,13 @@ unregister_connection(ClientId) when is_binary(ClientId) -> unregister_connection({ClientId, self()}); unregister_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:delete(?CONN_STATS_TAB, Conn), - _ = ets:delete(?CONN_ATTRS_TAB, Conn), - _ = ets:delete_object(?CONN_TAB, Conn), - notify({unregistered, ClientId, ConnPid}). + do_unregister_connection(Conn), + notify({unregistered, ConnPid}). + +do_unregister_connection(Conn) -> + true = ets:delete(?CONN_STATS_TAB, Conn), + true = ets:delete(?CONN_ATTRS_TAB, Conn), + true = ets:delete_object(?CONN_TAB, Conn). %% @doc Lookup connection pid -spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). @@ -138,7 +141,7 @@ handle_call(Req, _From, State) -> handle_cast({notify, {registered, ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:monitor(ConnPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> +handle_cast({notify, {unregistered, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:demonitor(ConnPid, PMon)}}; handle_cast(Msg, State) -> @@ -150,7 +153,12 @@ handle_info({'DOWN', _MRef, process, ConnPid, _Reason}, State = #{conn_pmon := P undefined -> {noreply, State}; ClientId -> - unregister_connection({ClientId, ConnPid}), + Conn = {ClientId, ConnPid}, + case ets:member(?CONN_ATTRS_TAB, Conn) of + true -> + ok = emqx_pool:async_submit(fun do_unregister_connection/1, [Conn]); + false -> ok + end, {noreply, State#{conn_pmon := emqx_pmon:erase(ConnPid, PMon)}} end; diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 573b913f7..3052191a1 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -849,14 +849,13 @@ shutdown(_Reason, #pstate{client_id = undefined}) -> ok; shutdown(_Reason, #pstate{connected = false}) -> ok; -shutdown(Reason, #pstate{client_id = ClientId}) when Reason =:= conflict; - Reason =:= discard -> - emqx_cm:unregister_connection(ClientId); -shutdown(Reason, PState = #pstate{connected = true, - client_id = ClientId}) -> +shutdown(conflict, _PState) -> + ok; +shutdown(discard, _PState) -> + ok; +shutdown(Reason, PState) -> ?LOG(info, "Shutdown for ~p", [Reason]), - emqx_hooks:run('client.disconnected', [credentials(PState), Reason]), - emqx_cm:unregister_connection(ClientId). + emqx_hooks:run('client.disconnected', [credentials(PState), Reason]). start_keepalive(0, _PState) -> ignore; diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 262a9a7a8..b92baa567 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -645,16 +645,14 @@ handle_info(Info, State) -> emqx_logger:error("[Session] unexpected info: ~p", [Info]), {noreply, State}. -terminate(Reason, #state{will_msg = WillMsg, client_id = ClientId, conn_pid = ConnPid}) -> - emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), +terminate(Reason, #state{will_msg = WillMsg, conn_pid = ConnPid}) -> + %% Should not run hooks here. + %% emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), send_willmsg(WillMsg), %% Ensure to shutdown the connection - if - ConnPid =/= undefined -> - ConnPid ! {shutdown, Reason}; - true -> ok - end, - emqx_sm:unregister_session(ClientId). + (ConnPid =:= undefined) orelse ConnPid ! {shutdown, Reason}. + %% Let it crash. + %% emqx_sm:unregister_session(ClientId). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -1011,3 +1009,4 @@ noreply(State) -> shutdown(Reason, State) -> {stop, {shutdown, Reason}, State}. + diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl index 644e33f37..8efc4afc8 100644 --- a/src/emqx_session_sup.erl +++ b/src/emqx_session_sup.erl @@ -38,7 +38,7 @@ init([]) -> [#{id => session, start => {emqx_session, start_link, []}, restart => temporary, - shutdown => 5000, + shutdown => brutal_kill, type => worker, modules => [emqx_session]}]}}. diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 637e44b0c..df2e4b862 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -40,9 +40,9 @@ -define(SM, ?MODULE). -%% ETS Tables --define(SESSION_TAB, emqx_session). --define(SESSION_P_TAB, emqx_persistent_session). +%% ETS Tables for session management. +-define(SESSION_TAB, emqx_session). +-define(SESSION_P_TAB, emqx_session_p). -define(SESSION_ATTRS_TAB, emqx_session_attrs). -define(SESSION_STATS_TAB, emqx_session_stats). @@ -59,8 +59,7 @@ open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid end, emqx_sm_locker:trans(ClientId, CleanStart); -open_session(SessAttrs = #{clean_start := false, - client_id := ClientId}) -> +open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> ResumeStart = fun(_) -> case resume_session(ClientId, SessAttrs) of {ok, SPid} -> @@ -77,13 +76,14 @@ discard_session(ClientId) when is_binary(ClientId) -> discard_session(ClientId, self()). discard_session(ClientId, ConnPid) when is_binary(ClientId) -> - lists:foreach(fun({_ClientId, SPid}) -> - case catch emqx_session:discard(SPid, ConnPid) of - {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> - emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); - ok -> ok - end - end, lookup_session(ClientId)). + lists:foreach( + fun({_ClientId, SPid}) -> + case catch emqx_session:discard(SPid, ConnPid) of + {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> + emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); + ok -> ok + end + end, lookup_session(ClientId)). %% @doc Try to resume a session. -spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}). @@ -116,19 +116,18 @@ close_session(SPid) when is_pid(SPid) -> register_session(ClientId, SessAttrs) when is_binary(ClientId) -> register_session({ClientId, self()}, SessAttrs); -register_session(Session = {ClientId, SPid}, SessAttrs) - when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_TAB, Session), - ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), - proplists:get_value(clean_start, SessAttrs, true) - andalso ets:insert(?SESSION_P_TAB, Session), - emqx_sm_registry:register_session(Session), +register_session(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) -> + true = ets:insert(?SESSION_TAB, Session), + true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), + true = proplists:get_value(clean_start, SessAttrs, true) + orelse ets:insert(?SESSION_P_TAB, Session), + ok = emqx_sm_registry:register_session(Session), notify({registered, ClientId, SPid}). %% @doc Get session attrs -spec(get_session_attrs({emqx_types:client_id(), pid()}) -> list(emqx_session:attr())). get_session_attrs(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_ATTRS_TAB, Session, []). + emqx_tables:lookup_value(?SESSION_ATTRS_TAB, Session, []). %% @doc Set session attrs -spec(set_session_attrs(emqx_types:client_id() | {emqx_types:client_id(), pid()}, @@ -144,17 +143,21 @@ unregister_session(ClientId) when is_binary(ClientId) -> unregister_session({ClientId, self()}); unregister_session(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - emqx_sm_registry:unregister_session(Session), - ets:delete(?SESSION_STATS_TAB, Session), - ets:delete(?SESSION_ATTRS_TAB, Session), - ets:delete_object(?SESSION_P_TAB, Session), - ets:delete_object(?SESSION_TAB, Session), + ok = do_unregister_session(Session), notify({unregistered, ClientId, SPid}). +%% @private +do_unregister_session(Session) -> + true = ets:delete(?SESSION_STATS_TAB, Session), + true = ets:delete(?SESSION_ATTRS_TAB, Session), + true = ets:delete_object(?SESSION_P_TAB, Session), + true = ets:delete_object(?SESSION_TAB, Session), + emqx_sm_registry:unregister_session(Session). + %% @doc Get session stats -spec(get_session_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). get_session_stats(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_STATS_TAB, Session, []). + emqx_tables:lookup_value(?SESSION_STATS_TAB, Session, []). %% @doc Set session stats -spec(set_session_stats(emqx_types:client_id() | {emqx_types:client_id(), pid()}, @@ -168,7 +171,7 @@ set_session_stats(Session = {ClientId, SPid}, Stats) when is_binary(ClientId), i -spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). lookup_session(ClientId) -> case emqx_sm_registry:is_enabled() of - true -> emqx_sm_registry:lookup_session(ClientId); + true -> emqx_sm_registry:lookup_session(ClientId); false -> ets:lookup(?SESSION_TAB, ClientId) end. @@ -185,13 +188,7 @@ dispatch(ClientId, Topic, Msg) -> %% @doc Lookup session pid. -spec(lookup_session_pid(emqx_types:client_id()) -> pid() | undefined). lookup_session_pid(ClientId) -> - safe_lookup_element(?SESSION_TAB, ClientId, undefined). - -safe_lookup_element(Tab, Key, Default) -> - try ets:lookup_element(Tab, Key, 2) - catch - error:badarg -> Default - end. + emqx_tables:lookup_value(?SESSION_TAB, ClientId). notify(Event) -> gen_server:cast(?SM, {notify, Event}). @@ -207,29 +204,34 @@ init([]) -> ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), ok = emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), - {ok, #{session_pmon => emqx_pmon:new()}}. + {ok, #{sess_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[SM] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({notify, {registered, ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; +handle_cast({notify, {registered, ClientId, SPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:demonitor(SPid, PMon)}}; +handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:demonitor(SPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{session_pmon := PMon}) -> +handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{sess_pmon := PMon}) -> case emqx_pmon:find(DownPid, PMon) of undefined -> {noreply, State}; - ClientId -> - unregister_session({ClientId, DownPid}), - {noreply, State#{session_pmon := emqx_pmon:erase(DownPid, PMon)}} + ClientId -> + Session = {ClientId, DownPid}, + case ets:member(?SESSION_ATTRS_TAB, Session) of + true -> + ok = emqx_pool:async_submit(fun do_unregister_session/1, [Session]); + false -> ok + end, + {noreply, State#{sess_pmon := emqx_pmon:erase(DownPid, PMon)}} end; handle_info(Info, State) -> diff --git a/src/emqx_sm_locker.erl b/src/emqx_sm_locker.erl index 29adf3342..409331b88 100644 --- a/src/emqx_sm_locker.erl +++ b/src/emqx_sm_locker.erl @@ -21,7 +21,7 @@ -export([trans/2, trans/3]). -export([lock/1, lock/2, unlock/1]). --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> ekka_locker:start_link(?MODULE). diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl index b503d71c8..3b472e2c8 100644 --- a/src/emqx_sm_registry.erl +++ b/src/emqx_sm_registry.erl @@ -41,8 +41,7 @@ start_link() -> gen_server:start_link({local, ?REGISTRY}, ?MODULE, [], []). -spec(is_enabled() -> boolean()). -is_enabled() -> - ets:info(?TAB, name) =/= undefined. +is_enabled() -> ets:info(?TAB, name) =/= undefined. -spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), session_pid()})). @@ -73,7 +72,7 @@ init([]) -> {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]), ok = ekka_mnesia:copy_table(?TAB), - _ = ekka:monitor(membership), + ok = ekka:monitor(membership), {ok, #{}}. handle_call(Req, _From, State) -> From d8cbf72da1b07bebefb613d61e36152126f7a1f3 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 12 Dec 2018 16:32:58 +0800 Subject: [PATCH 24/54] Fix 'badarg' crash --- src/emqx_session.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index b92baa567..4f0743210 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -648,11 +648,14 @@ handle_info(Info, State) -> terminate(Reason, #state{will_msg = WillMsg, conn_pid = ConnPid}) -> %% Should not run hooks here. %% emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), + %% Let it crash. + %% emqx_sm:unregister_session(ClientId), send_willmsg(WillMsg), %% Ensure to shutdown the connection - (ConnPid =:= undefined) orelse ConnPid ! {shutdown, Reason}. - %% Let it crash. - %% emqx_sm:unregister_session(ClientId). + if + ConnPid == undefined -> ok; + true -> ConnPid ! {shutdown, Reason} + end. code_change(_OldVsn, State, _Extra) -> {ok, State}. From 4e1d1bd60fb6506ca6c6f14fa9cc3886cef0864f Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 13 Dec 2018 14:13:13 +0800 Subject: [PATCH 25/54] Remove the emqx_session_sup module --- src/emqx_session_sup.erl | 44 ---------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 src/emqx_session_sup.erl diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl deleted file mode 100644 index 8efc4afc8..000000000 --- a/src/emqx_session_sup.erl +++ /dev/null @@ -1,44 +0,0 @@ -%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_session_sup). - --behavior(supervisor). - --include("emqx.hrl"). - --export([start_link/0, start_session/1]). - --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - --spec(start_session(map()) -> {ok, pid()}). -start_session(Attrs) -> - supervisor:start_child(?MODULE, [Attrs]). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, {{simple_one_for_one, 0, 1}, - [#{id => session, - start => {emqx_session, start_link, []}, - restart => temporary, - shutdown => brutal_kill, - type => worker, - modules => [emqx_session]}]}}. - From 4aaf0a7db498a2624e52b2677e387b59b44e9a95 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 13 Dec 2018 16:42:10 +0800 Subject: [PATCH 26/54] Remove emqx_session_sup to handle massive concurrent sessions --- src/emqx_session.erl | 39 +++++++++++++++++++++++++-------------- src/emqx_sm.erl | 4 ++-- src/emqx_sup.erl | 3 --- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 4f0743210..f0190dc14 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -370,8 +370,8 @@ init([Parent, #{zone := Zone, topic_alias_maximum = TopicAliasMaximum, will_msg = WillMsg }, - emqx_sm:register_session(ClientId, attrs(State)), - emqx_sm:set_session_stats(ClientId, stats(State)), + ok = emqx_sm:register_session(ClientId, attrs(State)), + true = emqx_sm:set_session_stats(ClientId, stats(State)), emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), ok = emqx_gc:init(GcPolicy), @@ -617,14 +617,20 @@ handle_info({timeout, Timer, emit_stats}, ?LOG(warning, "shutdown due to ~p", [Reason], NewState), shutdown(Reason, NewState) end; + handle_info({timeout, Timer, expired}, State = #state{expiry_timer = Timer}) -> - ?LOG(info, "expired, shutdown now:(", [], State), + ?LOG(info, "expired, shutdown now.", [], State), shutdown(expired, State); handle_info({timeout, Timer, will_delay}, State = #state{will_msg = WillMsg, will_delay_timer = Timer}) -> send_willmsg(WillMsg), {noreply, State#state{will_msg = undefined}}; +%% ConnPid is shutting down by the supervisor. +handle_info({'EXIT', ConnPid, Reason}, #state{conn_pid = ConnPid}) + when Reason =:= killed; Reason =:= shutdown -> + exit(Reason); + handle_info({'EXIT', ConnPid, Reason}, State = #state{will_msg = WillMsg, expiry_interval = 0, conn_pid = ConnPid}) -> send_willmsg(WillMsg), {stop, Reason, State#state{will_msg = undefined, conn_pid = undefined}}; @@ -641,30 +647,35 @@ handle_info({'EXIT', Pid, Reason}, State = #state{conn_pid = ConnPid}) -> ?LOG(error, "Unexpected EXIT: conn_pid=~p, exit_pid=~p, reason=~p", [ConnPid, Pid, Reason], State), {noreply, State}; + handle_info(Info, State) -> emqx_logger:error("[Session] unexpected info: ~p", [Info]), {noreply, State}. -terminate(Reason, #state{will_msg = WillMsg, conn_pid = ConnPid}) -> - %% Should not run hooks here. - %% emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), - %% Let it crash. - %% emqx_sm:unregister_session(ClientId), +terminate(Reason, #state{will_msg = WillMsg, + client_id = ClientId, + conn_pid = ConnPid, + old_conn_pid = OldConnPid}) -> send_willmsg(WillMsg), - %% Ensure to shutdown the connection - if - ConnPid == undefined -> ok; - true -> ConnPid ! {shutdown, Reason} - end. + [maybe_shutdown(Pid, Reason) || Pid <- [ConnPid, OldConnPid]], + emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]). code_change(_OldVsn, State, _Extra) -> {ok, State}. +maybe_shutdown(undefined, _Reason) -> + ok; +maybe_shutdown(Pid, normal) -> + Pid ! {shutdown, normal}; +maybe_shutdown(Pid, Reason) -> + exit(Pid, Reason). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ -has_connection(#state{conn_pid = Pid}) -> is_pid(Pid) andalso is_process_alive(Pid). +has_connection(#state{conn_pid = Pid}) -> + is_pid(Pid) andalso is_process_alive(Pid). handle_dispatch(Topic, Msg = #message{headers = Headers}, State = #state{subscriptions = SubMap, diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index df2e4b862..fd6a44231 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -55,7 +55,7 @@ start_link() -> open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid := ConnPid}) -> CleanStart = fun(_) -> ok = discard_session(ClientId, ConnPid), - emqx_session_sup:start_session(SessAttrs) + emqx_session:start_link(SessAttrs) end, emqx_sm_locker:trans(ClientId, CleanStart); @@ -65,7 +65,7 @@ open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> {ok, SPid} -> {ok, SPid, true}; {error, not_found} -> - emqx_session_sup:start_session(SessAttrs) + emqx_session:start_link(SessAttrs) end end, emqx_sm_locker:trans(ClientId, ResumeStart). diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 2f29dbfee..60be4db87 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -69,8 +69,6 @@ init([]) -> AccessControl = worker_spec(emqx_access_control), %% Session Manager SMSup = supervisor_spec(emqx_sm_sup), - %% Session Sup - SessionSup = supervisor_spec(emqx_session_sup), %% Connection Manager CMSup = supervisor_spec(emqx_cm_sup), %% Sys Sup @@ -83,7 +81,6 @@ init([]) -> BridgeSup, AccessControl, SMSup, - SessionSup, CMSup, SysSup]}}. From 7fe3d59c282ba2f591e8f2fa018e3313f9f170df Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 13 Dec 2018 17:25:50 +0800 Subject: [PATCH 27/54] Update the registered name of pool sup. --- src/emqx_broker_sup.erl | 4 ++-- src/emqx_pool_sup.erl | 3 ++- src/emqx_router_sup.erl | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl index a511e4154..f60a2a1b2 100644 --- a/src/emqx_broker_sup.erl +++ b/src/emqx_broker_sup.erl @@ -30,8 +30,8 @@ start_link() -> init([]) -> %% Broker pool PoolSize = emqx_vm:schedulers() * 2, - BrokerPool = emqx_pool_sup:spec(emqx_broker_pool, - [broker, hash, PoolSize, + BrokerPool = emqx_pool_sup:spec(broker_pool, + [emqx_broker_pool, hash, PoolSize, {emqx_broker, start_link, []}]), %% Shared subscription SharedSub = #{id => shared_sub, diff --git a/src/emqx_pool_sup.erl b/src/emqx_pool_sup.erl index b371549c0..e11fa01f2 100644 --- a/src/emqx_pool_sup.erl +++ b/src/emqx_pool_sup.erl @@ -37,7 +37,8 @@ spec(ChildId, Args) -> start_link(Pool, Type, MFA) -> start_link(Pool, Type, emqx_vm:schedulers(), MFA). --spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, term()}). +-spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) + -> {ok, pid()} | {error, term()}). start_link(Pool, Type, Size, MFA) when is_atom(Pool) -> supervisor:start_link({local, Pool}, ?MODULE, [Pool, Type, Size, MFA]); start_link(Pool, Type, Size, MFA) -> diff --git a/src/emqx_router_sup.erl b/src/emqx_router_sup.erl index 2bbaabc18..c0317fc78 100644 --- a/src/emqx_router_sup.erl +++ b/src/emqx_router_sup.erl @@ -32,8 +32,8 @@ init([]) -> modules => [emqx_router_helper]}, %% Router pool - RouterPool = emqx_pool_sup:spec(emqx_router_pool, - [router, hash, emqx_vm:schedulers(), + RouterPool = emqx_pool_sup:spec(router_pool, + [emqx_router_pool, hash, emqx_vm:schedulers(), {emqx_router, start_link, []}]), {ok, {{one_for_all, 0, 1}, [Helper, RouterPool]}}. From abe9aff06238343cb0ea496fa5fafdcc3adf491c Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 13 Dec 2018 18:12:32 +0800 Subject: [PATCH 28/54] Add 'enable_session_registry' config --- etc/emqx.conf | 5 +++++ priv/emqx.schema | 5 +++++ src/emqx_sm_registry.erl | 14 ++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 7876be59b..f38cc1a13 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1898,6 +1898,11 @@ plugins.expand_plugins_dir = {{ platform_plugins_dir }}/ ## Default: 1m, 1 minute broker.sys_interval = 1m +## Enable global session registry. +## +## Value: on | off +broker.enable_session_registry = on + ## Session locking strategy in a cluster. ## ## Value: Enum diff --git a/priv/emqx.schema b/priv/emqx.schema index 1001ab5a8..93b27f0ae 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1723,6 +1723,11 @@ end}. {default, "1m"} ]}. +{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ + {default, on}, + {datatype, flag} +]}. + {mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ {default, quorum}, {datatype, {enum, [local,one,quorum,all]}} diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl index 3b472e2c8..1d0df61bf 100644 --- a/src/emqx_sm_registry.erl +++ b/src/emqx_sm_registry.erl @@ -20,7 +20,6 @@ -export([start_link/0]). -export([is_enabled/0]). - -export([register_session/1, lookup_session/1, unregister_session/1]). %% gen_server callbacks @@ -41,7 +40,8 @@ start_link() -> gen_server:start_link({local, ?REGISTRY}, ?MODULE, [], []). -spec(is_enabled() -> boolean()). -is_enabled() -> ets:info(?TAB, name) =/= undefined. +is_enabled() -> + emqx_config:get_env(enable_session_registry, true). -spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), session_pid()})). @@ -50,11 +50,17 @@ lookup_session(ClientId) -> -spec(register_session({emqx_types:client_id(), session_pid()}) -> ok). register_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> - mnesia:dirty_write(?TAB, record(ClientId, SessPid)). + case is_enabled() of + true -> mnesia:dirty_write(?TAB, record(ClientId, SessPid)); + false -> ok + end. -spec(unregister_session({emqx_types:client_id(), session_pid()}) -> ok). unregister_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> - mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)). + case is_enabled() of + true -> mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)); + false -> ok + end. record(ClientId, SessPid) -> #global_session{sid = ClientId, pid = SessPid}. From d445c17e6c7fac4214262813187fcbd4e20ca5b3 Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Thu, 6 Dec 2018 01:18:55 +0800 Subject: [PATCH 29/54] Move some vm args to file vm.args --- Makefile | 2 +- etc/emqx.conf | 64 ----------------------------------- priv/emqx.schema | 88 ------------------------------------------------ 3 files changed, 1 insertion(+), 153 deletions(-) diff --git a/Makefile b/Makefile index 2c1693813..03bbef018 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ NO_AUTOPATCH = cuttlefish ERLC_OPTS += +debug_info -DAPPLICATION=emqx BUILD_DEPS = cuttlefish -dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.1.1 +dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.0 #TEST_DEPS = emqx_ct_helplers #dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers diff --git a/etc/emqx.conf b/etc/emqx.conf index f38cc1a13..19bd3f0e8 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -160,35 +160,6 @@ node.name = emqx@127.0.0.1 ## Value: String node.cookie = emqxsecretcookie -## Enable SMP support of Erlang VM. -## -## Value: enable | auto | disable -node.smp = auto - -## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable -## heartbeat, or set the value as 'on' -## -## Value: on -## -## vm.args: -heart -## node.heartbeat = on - -## Enable kernel poll. -## -## Value: on | off -## -## Default: on -node.kernel_poll = on - -## Sets the number of threads in async thread pool. Valid range is 0-1024. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: 0-1024 -## -## vm.args: +A Number -node.async_threads = 32 - ## Sets the maximum number of simultaneously existing processes for this ## system if a Number is passed as value. ## @@ -208,30 +179,6 @@ node.process_limit = 256000 ## vm.args: +Q Number node.max_ports = 256000 -## Set the distribution buffer busy limit (dist_buf_busy_limit). -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1KB-2GB] -## -## vm.args: +zdbbl size -node.dist_buffer_size = 8MB - -## Sets the maximum number of ETS tables. Note that mnesia and SSL will -## create temporary ETS tables. -## -## Value: Number -## -## vm.args: +e Number -node.max_ets_tables = 256000 - -## Tweak GC to run more often. -## -## Value: Number [0-65535] -## -## vm.args: -env ERL_FULLSWEEP_AFTER Number -node.fullsweep_after = 1000 - ## Crash dump log file. ## ## Value: Log file @@ -254,17 +201,6 @@ node.proto_dist = inet_tcp ## vm.args: -ssl_dist_optfile ## node.ssl_dist_optfile = {{ platform_etc_dir }}/ssl_dist.conf -## Sets the net_kernel tick time. TickTime is specified in seconds. -## Notice that all communicating nodes are to have the same TickTime -## value specified. -## -## See: http://www.erlang.org/doc/man/kernel_app.html#net_ticktime -## -## Value: Number -## -## vm.args: -kernel net_ticktime Number -node.dist_net_ticktime = 60 - ## Sets the port range for the listener socket of a distributed Erlang node. ## Note that if there are firewalls between clustered nodes, this port segment ## for nodes’ communication should be allowed. diff --git a/priv/emqx.schema b/priv/emqx.schema index 93b27f0ae..1b48681ab 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -191,40 +191,6 @@ end}. {default, "emqxsecretcookie"} ]}. -%% @doc SMP Support -{mapping, "node.smp", "vm_args.-smp", [ - {default, auto}, - {datatype, {enum, [enable, auto, disable]}}, - hidden -]}. - -%% @doc http://erlang.org/doc/man/heart.html -{mapping, "node.heartbeat", "vm_args.-heart", [ - {datatype, flag}, - hidden -]}. - -{translation, "vm_args.-heart", fun(Conf) -> - case cuttlefish:conf_get("node.heartbeat", Conf) of - true -> ""; - false -> cuttlefish:invalid("should be 'on' or comment the line!") - end -end}. - -%% @doc Enable Kernel Poll -{mapping, "node.kernel_poll", "vm_args.+K", [ - {default, on}, - {datatype, flag}, - hidden -]}. - -%% @doc More information at: http://erlang.org/doc/man/erl.html -{mapping, "node.async_threads", "vm_args.+A", [ - {default, 64}, - {datatype, integer}, - {validators, ["range:0-1024"]} -]}. - %% @doc Erlang Process Limit {mapping, "node.process_limit", "vm_args.+P", [ {datatype, integer}, @@ -245,53 +211,6 @@ end}. {validator, "range4ports", "must be 1024 to 134217727", fun(X) -> X >= 1024 andalso X =< 134217727 end}. -%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl -{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ - {datatype, bytesize}, - {commented, "32MB"}, - hidden, - {validators, ["zdbbl_range"]} -]}. - -{translation, "vm_args.+zdbbl", - fun(Conf) -> - ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), - case ZDBBL of - undefined -> undefined; - X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; - _ -> undefined - end - end -}. - -{validator, "zdbbl_range", "must be between 1KB and 2097151KB", - fun(ZDBBL) -> - %% 2097151KB = 2147482624 - ZDBBL >= 1024 andalso ZDBBL =< 2147482624 - end -}. - -%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 -{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ - {default, 1000}, - {datatype, integer}, - hidden, - {validators, ["positive_integer"]} -]}. - -{validator, "positive_integer", "must be a positive integer", - fun(X) -> X >= 0 end}. - -%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, -%% R16+ uses +e -%% @doc The ETS table limit -{mapping, "node.max_ets_tables", - cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ - {default, 256000}, - {datatype, integer}, - hidden -]}. - %% @doc Set the location of crash dumps {mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ {default, "{{crash_dump}}"}, @@ -299,13 +218,6 @@ end}. hidden ]}. -%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime -{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ - {commented, 60}, - {datatype, integer}, - hidden -]}. - %% @doc http://www.erlang.org/doc/man/kernel_app.html {mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ {commented, 6369}, From 14abbf96db64f8e21128785818d370b7783c0347 Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Thu, 6 Dec 2018 01:30:33 +0800 Subject: [PATCH 30/54] Update cuttlefish tag --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 424505d49..c3baebadb 100644 --- a/rebar.config +++ b/rebar.config @@ -9,7 +9,7 @@ {ekka, "v0.5.1"}, {clique, "develop"}, {esockd, "v5.4.2"}, - {cuttlefish, "v2.1.1"} + {cuttlefish, "v2.2.0"} ]}. {edoc_opts, [{preprocess, true}]}. From 68a6a88eb9b9924180a7ee7aaaa8e2f404e151ca Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Mon, 10 Dec 2018 16:58:04 +0800 Subject: [PATCH 31/54] Move all vm args into separate file vm.args --- etc/emqx.conf | 41 --------------------- etc/vm.args.cloud | 93 +++++++++++++++++++++++++++++++++++++++++++++++ etc/vm.args.edge | 93 +++++++++++++++++++++++++++++++++++++++++++++++ priv/emqx.schema | 40 -------------------- 4 files changed, 186 insertions(+), 81 deletions(-) create mode 100644 etc/vm.args.cloud create mode 100644 etc/vm.args.edge diff --git a/etc/emqx.conf b/etc/emqx.conf index 19bd3f0e8..45368a066 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -160,47 +160,6 @@ node.name = emqx@127.0.0.1 ## Value: String node.cookie = emqxsecretcookie -## Sets the maximum number of simultaneously existing processes for this -## system if a Number is passed as value. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1024-134217727] -## -## vm.args: +P Number -node.process_limit = 256000 - -## Sets the maximum number of simultaneously existing ports for this system. -## -## See: http://erlang.org/doc/man/erl.html -## -## Value: Number [1024-134217727] -## -## vm.args: +Q Number -node.max_ports = 256000 - -## Crash dump log file. -## -## Value: Log file -node.crash_dump = {{ platform_log_dir }}/crash.dump - -## Specify the erlang distributed protocol. -## -## Value: Enum -## - inet_tcp: the default; handles TCP streams with IPv4 addressing. -## - inet6_tcp: handles TCP with IPv6 addressing. -## - inet_tls: using TLS for Erlang Distribution. -## -## vm.args: -proto_dist inet_tcp -node.proto_dist = inet_tcp - -## Specify SSL Options in the file if using SSL for Erlang Distribution. -## -## Value: File -## -## vm.args: -ssl_dist_optfile -## node.ssl_dist_optfile = {{ platform_etc_dir }}/ssl_dist.conf - ## Sets the port range for the listener socket of a distributed Erlang node. ## Note that if there are firewalls between clustered nodes, this port segment ## for nodes’ communication should be allowed. diff --git a/etc/vm.args.cloud b/etc/vm.args.cloud new file mode 100644 index 000000000..47bf9d472 --- /dev/null +++ b/etc/vm.args.cloud @@ -0,0 +1,93 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## Basic args like '-name' and '-setcookie' should be configured in emqx.conf +## If they are configured in both file, the args configured in emqx.conf will +## be used. + +## Sets the maximum number of simultaneously existing processes for this system. ++P 256000 + +## Sets the maximum number of simultaneously existing ports for this system. ++Q 262144 + +## Sets the maximum number of ETS tables ++e 256000 + +## Sets the maximum number of atoms the virtual machine can handle. +#+t 1048576 + +## Set the location of crash dumps +-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +#-env ERL_FULLSWEEP_AFTER 1000 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). ++zdbbl 8192 + +## Sets default scheduler hint for port parallelism. ++spp true + +## Sets the number of threads in async thread pool. Valid range is 0-1024. ++A 8 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. +#+IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. +#+IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. +#+S 8:8 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. +#+SDcpu 8:8 + +## Sets the number of dirty I/O scheduler threads to create. +#+SDio 10 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/etc/vm.args.edge b/etc/vm.args.edge new file mode 100644 index 000000000..b6906e966 --- /dev/null +++ b/etc/vm.args.edge @@ -0,0 +1,93 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## Basic args like '-name' and '-setcookie' should be configured in emqx.conf +## If they are configured in both file, the args configured in emqx.conf will +## be used. + +## Sets the maximum number of simultaneously existing processes for this system. ++P 20480 + +## Sets the maximum number of simultaneously existing ports for this system. ++Q 4096 + +## Sets the maximum number of ETS tables ++e 512 + +## Sets the maximum number of atoms the virtual machine can handle. ++t 65536 + +## Set the location of crash dumps +-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +-env ERL_FULLSWEEP_AFTER 0 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). ++zdbbl 1024 + +## Sets default scheduler hint for port parallelism. ++spp false + +## Sets the number of threads in async thread pool. Valid range is 0-1024. ++A 1 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. ++IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. ++IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. ++S 1:1 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. ++SDcpu 1:1 + +## Sets the number of dirty I/O scheduler threads to create. ++SDio 1 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/priv/emqx.schema b/priv/emqx.schema index 1b48681ab..3bb9f34f8 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -173,51 +173,11 @@ end}. {default, "emqx@127.0.0.1"} ]}. -%% @doc The erlang distributed protocol -{mapping, "node.proto_dist", "vm_args.-proto_dist", [ - %{default, "inet_tcp"}, - {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, - hidden -]}. - -%% @doc Specify SSL Options in the file if using SSL for erlang distribution -{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ - {datatype, string}, - hidden -]}. - %% @doc Secret cookie for distributed erlang node {mapping, "node.cookie", "vm_args.-setcookie", [ {default, "emqxsecretcookie"} ]}. -%% @doc Erlang Process Limit -{mapping, "node.process_limit", "vm_args.+P", [ - {datatype, integer}, - {default, 256000}, - hidden -]}. - -%% Note: OTP R15 and earlier uses -env ERL_MAX_PORTS, R16+ uses +Q -%% @doc The number of concurrent ports/sockets -%% Valid range is 1024-134217727 -{mapping, "node.max_ports", - cuttlefish:otp("R16", "vm_args.+Q", "vm_args.-env ERL_MAX_PORTS"), [ - {default, 262144}, - {datatype, integer}, - {validators, ["range4ports"]} -]}. - -{validator, "range4ports", "must be 1024 to 134217727", - fun(X) -> X >= 1024 andalso X =< 134217727 end}. - -%% @doc Set the location of crash dumps -{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ - {default, "{{crash_dump}}"}, - {datatype, file}, - hidden -]}. - %% @doc http://www.erlang.org/doc/man/kernel_app.html {mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ {commented, 6369}, From 7c7d6b031c527dfff91995baa0dabae9d9524c10 Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Mon, 10 Dec 2018 18:08:24 +0800 Subject: [PATCH 32/54] Modify the NOTE descripition at the begining of the file --- etc/vm.args.cloud | 8 +++++--- etc/vm.args.edge | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/etc/vm.args.cloud b/etc/vm.args.cloud index 47bf9d472..d0992f470 100644 --- a/etc/vm.args.cloud +++ b/etc/vm.args.cloud @@ -3,9 +3,11 @@ ############################## ## NOTE: -## Basic args like '-name' and '-setcookie' should be configured in emqx.conf -## If they are configured in both file, the args configured in emqx.conf will -## be used. +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. ## Sets the maximum number of simultaneously existing processes for this system. +P 256000 diff --git a/etc/vm.args.edge b/etc/vm.args.edge index b6906e966..20adc41ab 100644 --- a/etc/vm.args.edge +++ b/etc/vm.args.edge @@ -3,9 +3,11 @@ ############################## ## NOTE: -## Basic args like '-name' and '-setcookie' should be configured in emqx.conf -## If they are configured in both file, the args configured in emqx.conf will -## be used. +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. ## Sets the maximum number of simultaneously existing processes for this system. +P 20480 From 3df8de2419cc302493d6e0ae9c705255c7d72b1f Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Fri, 14 Dec 2018 10:23:51 +0800 Subject: [PATCH 33/54] Rename vm.args.cloud -> vm.args --- etc/{vm.args.cloud => vm.args} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etc/{vm.args.cloud => vm.args} (100%) diff --git a/etc/vm.args.cloud b/etc/vm.args similarity index 100% rename from etc/vm.args.cloud rename to etc/vm.args From f54d414825654e1edf09363fb2602d5c9d0c97eb Mon Sep 17 00:00:00 2001 From: turtled Date: Sat, 15 Dec 2018 13:31:38 +0800 Subject: [PATCH 34/54] Fix pick fail --- src/emqx_broker.erl | 2 +- src/emqx_router.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 105589c65..60fb5fd8f 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -371,7 +371,7 @@ cast(Broker, Msg) -> %% Pick a broker pick(Topic) -> - gproc_pool:pick_worker(broker, Topic). + gproc_pool:pick_worker(emqx_broker_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks diff --git a/src/emqx_router.erl b/src/emqx_router.erl index c165b97ca..0951c82d6 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -148,7 +148,7 @@ call(Router, Msg) -> gen_server:call(Router, Msg, infinity). pick(Topic) -> - gproc_pool:pick_worker(router, Topic). + gproc_pool:pick_worker(emqx_router_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks From 721b72b96a25c7bdacf9350b8fe68deac0ee2b2c Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Mon, 17 Dec 2018 19:53:29 +0800 Subject: [PATCH 35/54] Add 'active_n' option to optimize the CPU usage of emqx_connection (#2060) * Add 'active_n' option to optimize the CPU usage of emqx_connection * Supports batch processing 'DOWN' events --- Makefile | 2 +- etc/emqx.conf | 15 +++ priv/emqx.schema | 11 ++ src/emqx_banned.erl | 44 +++---- src/emqx_broker.erl | 2 +- src/emqx_broker_helper.erl | 33 ++++-- src/emqx_broker_sup.erl | 4 +- src/emqx_cm.erl | 153 ++++++++++++------------ src/emqx_cm_sup.erl | 24 ++-- src/emqx_connection.erl | 55 +++++---- src/emqx_misc.erl | 18 +++ src/emqx_pmon.erl | 38 ++++-- src/emqx_pool_sup.erl | 2 - src/emqx_protocol.erl | 21 ++-- src/emqx_router.erl | 2 +- src/emqx_router_sup.erl | 4 +- src/emqx_session.erl | 5 +- src/emqx_sm.erl | 220 ++++++++++++++++++++--------------- src/emqx_sm_registry.erl | 5 +- test/emqx_banned_SUITE.erl | 29 +++-- test/emqx_cm_SUITE.erl | 22 ++-- test/emqx_pmon_SUITE.erl | 48 ++++++++ test/emqx_sequence_SUITE.erl | 2 +- test/emqx_sm_SUITE.erl | 16 ++- 24 files changed, 472 insertions(+), 303 deletions(-) create mode 100644 test/emqx_pmon_SUITE.erl diff --git a/Makefile b/Makefile index 03bbef018..68d383940 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ - emqx_hooks emqx_batch emqx_sequence + emqx_hooks emqx_batch emqx_sequence emqx_pmon CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/etc/emqx.conf b/etc/emqx.conf index 45368a066..09ce1b526 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -645,6 +645,11 @@ listener.tcp.external.max_connections = 1024000 ## Value: Number listener.tcp.external.max_conn_rate = 1000 +## Specify the {active, N} option for the external MQTT/TCP Socket. +## +## Value: Number +listener.tcp.external.active_n = 100 + ## Zone of the external MQTT/TCP listener belonged to. ## ## See: zone.$name.* @@ -781,6 +786,11 @@ listener.tcp.internal.max_connections = 10240000 ## Value: Number listener.tcp.internal.max_conn_rate = 1000 +## Specify the {active, N} option for the internal MQTT/TCP Socket. +## +## Value: Number +listener.tcp.internal.active_n = 1000 + ## Zone of the internal MQTT/TCP listener belonged to. ## ## Value: String @@ -888,6 +898,11 @@ listener.ssl.external.max_connections = 102400 ## Value: Number listener.ssl.external.max_conn_rate = 500 +## Specify the {active, N} option for the internal MQTT/SSL Socket. +## +## Value: Number +listener.ssl.external.active_n = 100 + ## Zone of the external MQTT/SSL listener belonged to. ## ## Value: String diff --git a/priv/emqx.schema b/priv/emqx.schema index 3bb9f34f8..7c37383c4 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -772,6 +772,11 @@ end}. {datatype, integer} ]}. +{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + {mapping, "listener.tcp.$name.zone", "emqx.listeners", [ {datatype, string} ]}. @@ -867,6 +872,11 @@ end}. {datatype, integer} ]}. +{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + {mapping, "listener.ssl.$name.zone", "emqx.listeners", [ {datatype, string} ]}. @@ -1283,6 +1293,7 @@ end}. {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, + {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, {rate_limit, Ratelimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, diff --git a/src/emqx_banned.erl b/src/emqx_banned.erl index 49a698384..af580b48e 100644 --- a/src/emqx_banned.erl +++ b/src/emqx_banned.erl @@ -26,17 +26,16 @@ -export([start_link/0]). -export([check/1]). --export([add/1, del/1]). +-export([add/1, delete/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(TAB, ?MODULE). --define(SERVER, ?MODULE). -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Mnesia bootstrap -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ mnesia(boot) -> ok = ekka_mnesia:create_table(?TAB, [ @@ -52,7 +51,7 @@ mnesia(copy) -> %% @doc Start the banned server. -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). -spec(check(emqx_types:credentials()) -> boolean()). check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) -> @@ -64,25 +63,25 @@ check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) - add(Banned) when is_record(Banned, banned) -> mnesia:dirty_write(?TAB, Banned). --spec(del({client_id, emqx_types:client_id()} | - {username, emqx_types:username()} | - {peername, emqx_types:peername()}) -> ok). -del(Key) -> +-spec(delete({client_id, emqx_types:client_id()} + | {username, emqx_types:username()} + | {peername, emqx_types:peername()}) -> ok). +delete(Key) -> mnesia:dirty_delete(?TAB, Key). -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> {ok, ensure_expiry_timer(#{expiry_timer => undefined})}. handle_call(Req, _From, State) -> - emqx_logger:error("[BANNED] unexpected call: ~p", [Req]), + emqx_logger:error("[Banned] unexpected call: ~p", [Req]), {reply, ignored, State}. handle_cast(Msg, State) -> - emqx_logger:error("[BANNED] unexpected msg: ~p", [Msg]), + emqx_logger:error("[Banned] unexpected msg: ~p", [Msg]), {noreply, State}. handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> @@ -90,7 +89,7 @@ handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> {noreply, ensure_expiry_timer(State), hibernate}; handle_info(Info, State) -> - emqx_logger:error("[BANNED] unexpected info: ~p", [Info]), + emqx_logger:error("[Banned] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> @@ -99,21 +98,22 @@ terminate(_Reason, #{expiry_timer := TRef}) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Internal functions -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ -ifdef(TEST). ensure_expiry_timer(State) -> State#{expiry_timer := emqx_misc:start_timer(timer:seconds(2), expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(5), expire)}. + State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> - mnesia:foldl(fun - (B = #banned{until = Until}, _Acc) when Until < Now -> - mnesia:delete_object(?TAB, B, sticky_write); - (_, _Acc) -> ok - end, ok, ?TAB). + mnesia:foldl( + fun(B = #banned{until = Until}, _Acc) when Until < Now -> + mnesia:delete_object(?TAB, B, sticky_write); + (_, _Acc) -> ok + end, ok, ?TAB). + diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 60fb5fd8f..6d1ee98cb 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -371,7 +371,7 @@ cast(Broker, Msg) -> %% Pick a broker pick(Topic) -> - gproc_pool:pick_worker(emqx_broker_pool, Topic). + gproc_pool:pick_worker(broker_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index 7d514e31d..c8e2b5eab 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -31,6 +31,8 @@ -define(SUBSEQ, emqx_subseq). -define(SHARD, 1024). +-define(BATCH_SIZE, 10000). + -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). @@ -106,14 +108,12 @@ handle_cast(Msg, State) -> emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #{pmon := PMon}) -> - case ets:lookup(?SUBMON, SubPid) of - [{_, SubId}] -> - ok = emqx_pool:async_submit(fun subscriber_down/2, [SubPid, SubId]); - [] -> - emqx_logger:error("[BrokerHelper] unexpected DOWN: ~p, reason: ~p", [SubPid, Reason]) - end, - {noreply, State#{pmon := emqx_pmon:erase(SubPid, PMon)}}; +handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> + SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, SubPids]), + {_, PMon1} = emqx_pmon:erase_all(SubPids, PMon), + {noreply, State#{pmon := PMon1}}; handle_info(Info, State) -> emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), @@ -126,8 +126,17 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -subscriber_down(SubPid, SubId) -> - true = ets:delete(?SUBMON, SubPid), - true = (SubId =:= undefined) orelse ets:delete_object(?SUBID, {SubId, SubPid}), - emqx_broker:subscriber_down(SubPid). +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +clean_down(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [{_, SubId}] -> + true = ets:delete(?SUBMON, SubPid), + true = (SubId =:= undefined) + orelse ets:delete_object(?SUBID, {SubId, SubPid}), + emqx_broker:subscriber_down(SubPid); + [] -> ok + end. diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl index f60a2a1b2..5b1c0a0e7 100644 --- a/src/emqx_broker_sup.erl +++ b/src/emqx_broker_sup.erl @@ -30,9 +30,9 @@ start_link() -> init([]) -> %% Broker pool PoolSize = emqx_vm:schedulers() * 2, - BrokerPool = emqx_pool_sup:spec(broker_pool, - [emqx_broker_pool, hash, PoolSize, + BrokerPool = emqx_pool_sup:spec([broker_pool, hash, PoolSize, {emqx_broker, start_link, []}]), + %% Shared subscription SharedSub = #{id => shared_sub, start => {emqx_shared_sub, start_link, []}, diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 0d2ecf5eb..62cb417de 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -20,74 +20,57 @@ -export([start_link/0]). --export([lookup_connection/1]). -export([register_connection/1, register_connection/2]). --export([unregister_connection/1]). --export([get_conn_attrs/1, set_conn_attrs/2]). --export([get_conn_stats/1, set_conn_stats/2]). +-export([unregister_connection/1, unregister_connection/2]). +-export([get_conn_attrs/1, get_conn_attrs/2]). +-export([set_conn_attrs/2, set_conn_attrs/3]). +-export([get_conn_stats/1, get_conn_stats/2]). +-export([set_conn_stats/2, set_conn_stats/3]). -export([lookup_conn_pid/1]). +%% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). %% internal export --export([update_conn_stats/0]). +-export([stats_fun/0]). -define(CM, ?MODULE). -%% ETS Tables. +%% ETS tables for connection management. -define(CONN_TAB, emqx_conn). -define(CONN_ATTRS_TAB, emqx_conn_attrs). -define(CONN_STATS_TAB, emqx_conn_stats). +-define(BATCH_SIZE, 10000). + %% @doc Start the connection manager. -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?CM}, ?MODULE, [], []). -%% @doc Lookup a connection. --spec(lookup_connection(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). -lookup_connection(ClientId) when is_binary(ClientId) -> - ets:lookup(?CONN_TAB, ClientId). +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ %% @doc Register a connection. --spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). +-spec(register_connection(emqx_types:client_id()) -> ok). register_connection(ClientId) when is_binary(ClientId) -> - register_connection({ClientId, self()}); + register_connection(ClientId, self()). -register_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - true = ets:insert(?CONN_TAB, Conn), +-spec(register_connection(emqx_types:client_id(), pid()) -> ok). +register_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = ets:insert(?CONN_TAB, {ClientId, ConnPid}), notify({registered, ClientId, ConnPid}). --spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}, list()) -> ok). -register_connection(ClientId, Attrs) when is_binary(ClientId) -> - register_connection({ClientId, self()}, Attrs); -register_connection(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> - set_conn_attrs(Conn, Attrs), - register_connection(Conn). - -%% @doc Get conn attrs --spec(get_conn_attrs({emqx_types:client_id(), pid()}) -> list()). -get_conn_attrs(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - try - ets:lookup_element(?CONN_ATTRS_TAB, Conn, 2) - catch - error:badarg -> [] - end. - -%% @doc Set conn attrs -set_conn_attrs(ClientId, Attrs) when is_binary(ClientId) -> - set_conn_attrs({ClientId, self()}, Attrs); -set_conn_attrs(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> - ets:insert(?CONN_ATTRS_TAB, {Conn, Attrs}). - -%% @doc Unregister a conn. --spec(unregister_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). +%% @doc Unregister a connection. +-spec(unregister_connection(emqx_types:client_id()) -> ok). unregister_connection(ClientId) when is_binary(ClientId) -> - unregister_connection({ClientId, self()}); + unregister_connection(ClientId, self()). -unregister_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - do_unregister_connection(Conn), +-spec(unregister_connection(emqx_types:client_id(), pid()) -> ok). +unregister_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = do_unregister_connection({ClientId, ConnPid}), notify({unregistered, ConnPid}). do_unregister_connection(Conn) -> @@ -95,30 +78,52 @@ do_unregister_connection(Conn) -> true = ets:delete(?CONN_ATTRS_TAB, Conn), true = ets:delete_object(?CONN_TAB, Conn). -%% @doc Lookup connection pid --spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). -lookup_conn_pid(ClientId) when is_binary(ClientId) -> - case ets:lookup(?CONN_TAB, ClientId) of - [] -> undefined; - [{_, Pid}] -> Pid - end. +%% @doc Get conn attrs +-spec(get_conn_attrs(emqx_types:client_id()) -> list()). +get_conn_attrs(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_attrs(ClientId, ConnPid). + +-spec(get_conn_attrs(emqx_types:client_id(), pid()) -> list()). +get_conn_attrs(ClientId, ConnPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_ATTRS_TAB, {ClientId, ConnPid}, []). + +%% @doc Set conn attrs +-spec(set_conn_attrs(emqx_types:client_id(), list()) -> true). +set_conn_attrs(ClientId, Attrs) when is_binary(ClientId) -> + set_conn_attrs(ClientId, self(), Attrs). + +-spec(set_conn_attrs(emqx_types:client_id(), pid(), list()) -> true). +set_conn_attrs(ClientId, ConnPid, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, + ets:insert(?CONN_ATTRS_TAB, {Conn, Attrs}). %% @doc Get conn stats --spec(get_conn_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). -get_conn_stats(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - try ets:lookup_element(?CONN_STATS_TAB, Conn, 2) - catch - error:badarg -> [] - end. +-spec(get_conn_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_stats(ClientId, ConnPid). + +-spec(get_conn_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId, ConnPid) when is_binary(ClientId) -> + Conn = {ClientId, ConnPid}, + emqx_tables:lookup_value(?CONN_STATS_TAB, Conn, []). %% @doc Set conn stats. --spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> boolean()). +-spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> true). set_conn_stats(ClientId, Stats) when is_binary(ClientId) -> - set_conn_stats({ClientId, self()}, Stats); + set_conn_stats(ClientId, self(), Stats). -set_conn_stats(Conn = {ClientId, ConnPid}, Stats) when is_binary(ClientId), is_pid(ConnPid) -> +-spec(set_conn_stats(emqx_types:client_id(), pid(), list(emqx_stats:stats())) -> true). +set_conn_stats(ClientId, ConnPid, Stats) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, ets:insert(?CONN_STATS_TAB, {Conn, Stats}). +%% @doc Lookup connection pid. +-spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). +lookup_conn_pid(ClientId) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_TAB, ClientId). + notify(Msg) -> gen_server:cast(?CM, {notify, Msg}). @@ -131,7 +136,7 @@ init([]) -> ok = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), ok = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?CONN_STATS_TAB, TabOpts), - ok = emqx_stats:update_interval(cm_stats, fun ?MODULE:update_conn_stats/0), + ok = emqx_stats:update_interval(conn_stats, fun ?MODULE:stats_fun/0), {ok, #{conn_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> @@ -148,26 +153,19 @@ handle_cast(Msg, State) -> emqx_logger:error("[CM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, ConnPid, _Reason}, State = #{conn_pmon := PMon}) -> - case emqx_pmon:find(ConnPid, PMon) of - undefined -> - {noreply, State}; - ClientId -> - Conn = {ClientId, ConnPid}, - case ets:member(?CONN_ATTRS_TAB, Conn) of - true -> - ok = emqx_pool:async_submit(fun do_unregister_connection/1, [Conn]); - false -> ok - end, - {noreply, State#{conn_pmon := emqx_pmon:erase(ConnPid, PMon)}} - end; +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{conn_pmon := PMon}) -> + ConnPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ConnPids, PMon), + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{conn_pmon := PMon1}}; handle_info(Info, State) -> emqx_logger:error("[CM] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(cm_stats). + emqx_stats:cancel_update(conn_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -176,7 +174,16 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -update_conn_stats() -> +clean_down({Pid, ClientId}) -> + Conn = {ClientId, Pid}, + case ets:member(?CONN_TAB, ClientId) + orelse ets:member(?CONN_ATTRS_TAB, Conn) of + true -> + do_unregister_connection(Conn); + false -> false + end. + +stats_fun() -> case ets:info(?CONN_TAB, size) of undefined -> ok; Size -> emqx_stats:setstat('connections/count', 'connections/max', Size) diff --git a/src/emqx_cm_sup.erl b/src/emqx_cm_sup.erl index 000e79336..19dd9cb50 100644 --- a/src/emqx_cm_sup.erl +++ b/src/emqx_cm_sup.erl @@ -25,17 +25,17 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Banned = #{id => banned, - start => {emqx_banned, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_banned]}, - Manager = #{id => manager, - start => {emqx_cm, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm]}, + Banned = #{id => banned, + start => {emqx_banned, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [emqx_banned]}, + Manager = #{id => manager, + start => {emqx_cm, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_cm]}, {ok, {{one_for_one, 10, 100}, [Banned, Manager]}}. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 3d0ab0df7..e1d4011fc 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -38,7 +38,7 @@ peername, sockname, conn_state, - await_recv, + active_n, proto_state, parser_state, keepalive, @@ -51,6 +51,7 @@ idle_timeout }). +-define(DEFAULT_ACTIVE_N, 100). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). start_link(Transport, Socket, Options) -> @@ -69,7 +70,7 @@ info(#state{transport = Transport, peername = Peername, sockname = Sockname, conn_state = ConnState, - await_recv = AwaitRecv, + active_n = ActiveN, rate_limit = RateLimit, publish_limit = PubLimit, proto_state = ProtoState}) -> @@ -77,7 +78,7 @@ info(#state{transport = Transport, {peername, Peername}, {sockname, Sockname}, {conn_state, ConnState}, - {await_recv, AwaitRecv}, + {active_n, ActiveN}, {rate_limit, esockd_rate_limit:info(RateLimit)}, {publish_limit, esockd_rate_limit:info(PubLimit)}], ProtoInfo = emqx_protocol:info(ProtoState), @@ -87,8 +88,8 @@ info(#state{transport = Transport, attrs(CPid) when is_pid(CPid) -> call(CPid, attrs); -attrs(#state{peername = Peername, - sockname = Sockname, +attrs(#state{peername = Peername, + sockname = Sockname, proto_state = ProtoState}) -> SockAttrs = [{peername, Peername}, {sockname, Sockname}], @@ -129,6 +130,7 @@ init([Transport, RawSocket, Options]) -> Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), RateLimit = init_limiter(proplists:get_value(rate_limit, Options)), PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)), + ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N), EnableStats = emqx_zone:get_env(Zone, enable_stats, true), IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), SendFun = send_fun(Transport, Socket), @@ -140,8 +142,8 @@ init([Transport, RawSocket, Options]) -> State = run_socket(#state{transport = Transport, socket = Socket, peername = Peername, - await_recv = false, conn_state = running, + active_n = ActiveN, rate_limit = RateLimit, publish_limit = PubLimit, proto_state = ProtoState, @@ -243,19 +245,26 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), shutdown(conflict, State); -handle_info(activate_sock, State) -> - {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; - -handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> +handle_info({tcp, _Sock, Data}, State) -> ?LOG(debug, "RECV ~p", [Data]), Size = iolist_size(Data), emqx_metrics:trans(inc, 'bytes/received', Size), Incoming = #{bytes => Size, packets => 0}, - handle_packet(Data, State#state{await_recv = false, incoming = Incoming}); + handle_packet(Data, State#state{incoming = Incoming}); -handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> +%% Rate limit here, cool:) +handle_info({tcp_passive, _Sock}, State) -> + {noreply, ensure_rate_limit(State)}; + +handle_info({tcp_error, _Sock, Reason}, State) -> shutdown(Reason, State); +handle_info({tcp_closed, _Sock}, State) -> + shutdown(closed, State); + +handle_info(activate_sock, State) -> + {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; + handle_info({inet_reply, _Sock, ok}, State) -> {noreply, State}; @@ -314,16 +323,17 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ %% Receive and parse data -handle_packet(<<>>, State0) -> - State = ensure_stats_timer(ensure_rate_limit(State0)), - ok = maybe_gc(State, incoming), - {noreply, State}; +handle_packet(<<>>, State) -> + NState = ensure_stats_timer(State), + ok = maybe_gc(NState, incoming), + {noreply, NState}; + handle_packet(Data, State = #state{proto_state = ProtoState, parser_state = ParserState, idle_timeout = IdleTimeout}) -> case catch emqx_frame:parse(Data, ParserState) of {more, NewParserState} -> - {noreply, run_socket(State#state{parser_state = NewParserState}), IdleTimeout}; + {noreply, State#state{parser_state = NewParserState}, IdleTimeout}; {ok, Packet = ?PACKET(Type), Rest} -> emqx_metrics:received(Packet), case emqx_protocol:received(Packet, ProtoState) of @@ -352,6 +362,7 @@ reset_parser(State = #state{proto_state = ProtoState}) -> inc_publish_cnt(Type, State = #state{incoming = Incoming = #{packets := Cnt}}) when Type == ?PUBLISH; Type == ?SUBSCRIBE -> State#state{incoming = Incoming#{packets := Cnt + 1}}; + inc_publish_cnt(_Type, State) -> State. @@ -379,11 +390,11 @@ ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) -> run_socket(State = #state{conn_state = blocked}) -> State; -run_socket(State = #state{await_recv = true}) -> - State; -run_socket(State = #state{transport = Transport, socket = Socket}) -> - Transport:async_recv(Socket, 0, infinity), - State#state{await_recv = true}. +run_socket(State = #state{transport = Transport, + socket = Socket, + active_n = ActiveN}) -> + Transport:setopts(Socket, [{active, ActiveN}]), + State. %%------------------------------------------------------------------------------ %% Ensure stats timer diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index cf4a555ca..38bc1c2b0 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -19,6 +19,8 @@ -export([init_proc_mng_policy/1, conn_proc_mng_policy/1]). +-export([drain_down/1]). + %% @doc Merge options -spec(merge_opts(list(), list()) -> list()). merge_opts(Defaults, Options) -> @@ -108,3 +110,19 @@ is_enabled(Max) -> is_integer(Max) andalso Max > ?DISABLED. proc_info(Key) -> {Key, Value} = erlang:process_info(self(), Key), Value. + +-spec(drain_down(pos_integer()) -> list(pid())). +drain_down(Cnt) when Cnt > 0 -> + drain_down(Cnt, []). + +drain_down(0, Acc) -> + lists:reverse(Acc); + +drain_down(Cnt, Acc) -> + receive + {'DOWN', _MRef, process, Pid, _Reason} -> + drain_down(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + diff --git a/src/emqx_pmon.erl b/src/emqx_pmon.erl index 9b874041e..a00212ed5 100644 --- a/src/emqx_pmon.erl +++ b/src/emqx_pmon.erl @@ -14,24 +14,27 @@ -module(emqx_pmon). +-compile({no_auto_import, [monitor/3]}). + -export([new/0]). -export([monitor/2, monitor/3]). -export([demonitor/2]). -export([find/2]). --export([erase/2]). - --compile({no_auto_import,[monitor/3]}). +-export([erase/2, erase_all/2]). +-export([count/1]). -type(pmon() :: {?MODULE, map()}). -export_type([pmon/0]). -spec(new() -> pmon()). -new() -> {?MODULE, maps:new()}. +new() -> + {?MODULE, maps:new()}. -spec(monitor(pid(), pmon()) -> pmon()). monitor(Pid, PM) -> - monitor(Pid, undefined, PM). + ?MODULE:monitor(Pid, undefined, PM). +-spec(monitor(pid(), term(), pmon()) -> pmon()). monitor(Pid, Val, {?MODULE, PM}) -> {?MODULE, case maps:is_key(Pid, PM) of true -> PM; @@ -43,21 +46,36 @@ monitor(Pid, Val, {?MODULE, PM}) -> demonitor(Pid, {?MODULE, PM}) -> {?MODULE, case maps:find(Pid, PM) of {ok, {Ref, _Val}} -> - %% Don't flush - _ = erlang:demonitor(Ref), + %% flush + _ = erlang:demonitor(Ref, [flush]), maps:remove(Pid, PM); error -> PM end}. --spec(find(pid(), pmon()) -> undefined | term()). +-spec(find(pid(), pmon()) -> error | {ok, term()}). find(Pid, {?MODULE, PM}) -> case maps:find(Pid, PM) of {ok, {_Ref, Val}} -> - Val; - error -> undefined + {ok, Val}; + error -> error end. -spec(erase(pid(), pmon()) -> pmon()). erase(Pid, {?MODULE, PM}) -> {?MODULE, maps:remove(Pid, PM)}. +-spec(erase_all([pid()], pmon()) -> {[{pid(), term()}], pmon()}). +erase_all(Pids, PMon0) -> + lists:foldl( + fun(Pid, {Acc, PMon}) -> + case find(Pid, PMon) of + {ok, Val} -> + {[{Pid, Val}|Acc], erase(Pid, PMon)}; + error -> {Acc, PMon} + end + end, {[], PMon0}, Pids). + +-spec(count(pmon()) -> non_neg_integer()). +count({?MODULE, PM}) -> + maps:size(PM). + diff --git a/src/emqx_pool_sup.erl b/src/emqx_pool_sup.erl index e11fa01f2..eb81233f9 100644 --- a/src/emqx_pool_sup.erl +++ b/src/emqx_pool_sup.erl @@ -39,8 +39,6 @@ start_link(Pool, Type, MFA) -> -spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, term()}). -start_link(Pool, Type, Size, MFA) when is_atom(Pool) -> - supervisor:start_link({local, Pool}, ?MODULE, [Pool, Type, Size, MFA]); start_link(Pool, Type, Size, MFA) -> supervisor:start_link(?MODULE, [Pool, Type, Size, MFA]). diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 3052191a1..dad40e190 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -320,7 +320,8 @@ process_packet(?CONNECT_PACKET( case try_open_session(PState3) of {ok, SPid, SP} -> PState4 = PState3#pstate{session = SPid, connected = true}, - ok = emqx_cm:register_connection(client_id(PState4), attrs(PState4)), + ok = emqx_cm:register_connection(client_id(PState4)), + true = emqx_cm:set_conn_attrs(client_id(PState4), attrs(PState4)), %% Start keepalive start_keepalive(Keepalive, PState4), %% Success @@ -497,18 +498,18 @@ do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId), puback(?QOS_0, _PacketId, _Result, PState) -> {ok, PState}; +puback(?QOS_1, PacketId, [], PState) -> + deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); +puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({puback, PacketId, ?RC_SUCCESS}, PState); puback(?QOS_1, PacketId, {error, ReasonCode}, PState) -> deliver({puback, PacketId, ReasonCode}, PState); -puback(?QOS_1, PacketId, {ok, []}, PState) -> - deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_1, PacketId, {ok, _}, PState) -> - deliver({puback, PacketId, ?RC_SUCCESS}, PState); -puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> - deliver({pubrec, PacketId, ReasonCode}, PState); -puback(?QOS_2, PacketId, {ok, []}, PState) -> +puback(?QOS_2, PacketId, [], PState) -> deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_2, PacketId, {ok, _}, PState) -> - deliver({pubrec, PacketId, ?RC_SUCCESS}, PState). +puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({pubrec, PacketId, ?RC_SUCCESS}, PState); +puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> + deliver({pubrec, PacketId, ReasonCode}, PState). %%------------------------------------------------------------------------------ %% Deliver Packet -> Client diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 0951c82d6..0463526d6 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -148,7 +148,7 @@ call(Router, Msg) -> gen_server:call(Router, Msg, infinity). pick(Topic) -> - gproc_pool:pick_worker(emqx_router_pool, Topic). + gproc_pool:pick_worker(router_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks diff --git a/src/emqx_router_sup.erl b/src/emqx_router_sup.erl index c0317fc78..945d7910f 100644 --- a/src/emqx_router_sup.erl +++ b/src/emqx_router_sup.erl @@ -17,6 +17,7 @@ -behaviour(supervisor). -export([start_link/0]). + -export([init/1]). start_link() -> @@ -32,8 +33,7 @@ init([]) -> modules => [emqx_router_helper]}, %% Router pool - RouterPool = emqx_pool_sup:spec(router_pool, - [emqx_router_pool, hash, emqx_vm:schedulers(), + RouterPool = emqx_pool_sup:spec([router_pool, hash, {emqx_router, start_link, []}]), {ok, {{one_for_all, 0, 1}, [Helper, RouterPool]}}. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index f0190dc14..f41a25fc5 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -259,7 +259,7 @@ subscribe(SPid, PacketId, Properties, TopicFilters) -> %% @doc Called by connection processes when publishing messages -spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message()) - -> {ok, emqx_types:deliver_results()}). + -> emqx_types:deliver_results() | {error, term()}). publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) -> %% Publish QoS0 message directly emqx_broker:publish(Msg); @@ -370,7 +370,8 @@ init([Parent, #{zone := Zone, topic_alias_maximum = TopicAliasMaximum, will_msg = WillMsg }, - ok = emqx_sm:register_session(ClientId, attrs(State)), + ok = emqx_sm:register_session(ClientId, self()), + true = emqx_sm:set_session_attrs(ClientId, attrs(State)), true = emqx_sm:set_session_stats(ClientId, stats(State)), emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index fd6a44231..281893fb0 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -21,12 +21,15 @@ -export([start_link/0]). -export([open_session/1, close_session/1]). --export([lookup_session/1, lookup_session_pid/1]). -export([resume_session/2]). -export([discard_session/1, discard_session/2]). --export([register_session/2, unregister_session/1]). --export([get_session_attrs/1, set_session_attrs/2]). --export([get_session_stats/1, set_session_stats/2]). +-export([register_session/1, register_session/2]). +-export([unregister_session/1, unregister_session/2]). +-export([get_session_attrs/1, get_session_attrs/2, + set_session_attrs/2, set_session_attrs/3]). +-export([get_session_stats/1, get_session_stats/2, + set_session_stats/2, set_session_stats/3]). +-export([lookup_session_pids/1]). %% Internal functions for rpc -export([dispatch/3]). @@ -46,6 +49,8 @@ -define(SESSION_ATTRS_TAB, emqx_session_attrs). -define(SESSION_STATS_TAB, emqx_session_stats). +-define(BATCH_SIZE, 10000). + -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?SM}, ?MODULE, [], []). @@ -62,8 +67,8 @@ open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> ResumeStart = fun(_) -> case resume_session(ClientId, SessAttrs) of - {ok, SPid} -> - {ok, SPid, true}; + {ok, SessPid} -> + {ok, SessPid, true}; {error, not_found} -> emqx_session:start_link(SessAttrs) end @@ -75,76 +80,68 @@ open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> discard_session(ClientId) when is_binary(ClientId) -> discard_session(ClientId, self()). +-spec(discard_session(emqx_types:client_id(), pid()) -> ok). discard_session(ClientId, ConnPid) when is_binary(ClientId) -> lists:foreach( - fun({_ClientId, SPid}) -> - case catch emqx_session:discard(SPid, ConnPid) of - {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> - emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); - ok -> ok - end - end, lookup_session(ClientId)). + fun(SessPid) -> + try emqx_session:discard(SessPid, ConnPid) + catch + _:Error:_Stk -> + emqx_logger:error("[SM] Failed to discard ~p: ~p", [SessPid, Error]) + end + end, lookup_session_pids(ClientId)). %% @doc Try to resume a session. -spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}). resume_session(ClientId, SessAttrs = #{conn_pid := ConnPid}) -> - case lookup_session(ClientId) of + case lookup_session_pids(ClientId) of [] -> {error, not_found}; - [{_ClientId, SPid}] -> - ok = emqx_session:resume(SPid, SessAttrs), - {ok, SPid}; - Sessions -> - [{_, SPid}|StaleSessions] = lists:reverse(Sessions), - emqx_logger:error("[SM] More than one session found: ~p", [Sessions]), - lists:foreach(fun({_, StalePid}) -> + [SessPid] -> + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid}; + SessPids -> + [SessPid|StalePids] = lists:reverse(SessPids), + emqx_logger:error("[SM] More than one session found: ~p", [SessPids]), + lists:foreach(fun(StalePid) -> catch emqx_session:discard(StalePid, ConnPid) - end, StaleSessions), - ok = emqx_session:resume(SPid, SessAttrs), - {ok, SPid} + end, StalePids), + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid} end. %% @doc Close a session. --spec(close_session({emqx_types:client_id(), pid()} | pid()) -> ok). -close_session({_ClientId, SPid}) -> - emqx_session:close(SPid); -close_session(SPid) when is_pid(SPid) -> - emqx_session:close(SPid). +-spec(close_session(emqx_types:client_id() | pid()) -> ok). +close_session(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> ok; + [SessPid] -> close_session(SessPid); + SessPids -> lists:foreach(fun close_session/1, SessPids) + end; -%% @doc Register a session with attributes. --spec(register_session(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - list(emqx_session:attr())) -> ok). -register_session(ClientId, SessAttrs) when is_binary(ClientId) -> - register_session({ClientId, self()}, SessAttrs); +close_session(SessPid) when is_pid(SessPid) -> + emqx_session:close(SessPid). -register_session(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) -> +%% @doc Register a session. +-spec(register_session(emqx_types:client_id()) -> ok). +register_session(ClientId) when is_binary(ClientId) -> + register_session(ClientId, self()). + +-spec(register_session(emqx_types:client_id(), pid()) -> ok). +register_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, true = ets:insert(?SESSION_TAB, Session), - true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), - true = proplists:get_value(clean_start, SessAttrs, true) - orelse ets:insert(?SESSION_P_TAB, Session), ok = emqx_sm_registry:register_session(Session), - notify({registered, ClientId, SPid}). - -%% @doc Get session attrs --spec(get_session_attrs({emqx_types:client_id(), pid()}) -> list(emqx_session:attr())). -get_session_attrs(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - emqx_tables:lookup_value(?SESSION_ATTRS_TAB, Session, []). - -%% @doc Set session attrs --spec(set_session_attrs(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - list(emqx_session:attr())) -> true). -set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) -> - set_session_attrs({ClientId, self()}, SessAttrs); -set_session_attrs(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}). + notify({registered, ClientId, SessPid}). %% @doc Unregister a session --spec(unregister_session(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). +-spec(unregister_session(emqx_types:client_id()) -> ok). unregister_session(ClientId) when is_binary(ClientId) -> - unregister_session({ClientId, self()}); + unregister_session(ClientId, self()). -unregister_session(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - ok = do_unregister_session(Session), - notify({unregistered, ClientId, SPid}). +-spec(unregister_session(emqx_types:client_id(), pid()) -> ok). +unregister_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + ok = do_unregister_session({ClientId, SessPid}), + notify({unregistered, SessPid}). %% @private do_unregister_session(Session) -> @@ -154,42 +151,69 @@ do_unregister_session(Session) -> true = ets:delete_object(?SESSION_TAB, Session), emqx_sm_registry:unregister_session(Session). +%% @doc Get session attrs +-spec(get_session_attrs(emqx_types:client_id()) -> list(emqx_session:attr())). +get_session_attrs(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> get_session_attrs(ClientId, SessPid) + end. + +-spec(get_session_attrs(emqx_types:client_id(), pid()) -> list(emqx_session:attr())). +get_session_attrs(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + emqx_tables:lookup_value(?SESSION_ATTRS_TAB, {ClientId, SessPid}, []). + +%% @doc Set session attrs +-spec(set_session_attrs(emqx_types:client_id(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) -> + set_session_attrs(ClientId, self(), SessAttrs). + +-spec(set_session_attrs(emqx_types:client_id(), pid(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessPid, SessAttrs) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), + proplists:get_value(clean_start, SessAttrs, true) orelse ets:insert(?SESSION_P_TAB, Session). + %% @doc Get session stats --spec(get_session_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). -get_session_stats(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - emqx_tables:lookup_value(?SESSION_STATS_TAB, Session, []). +-spec(get_session_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_session_stats(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> + get_session_stats(ClientId, SessPid) + end. + +-spec(get_session_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_session_stats(ClientId, SessPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?SESSION_STATS_TAB, {ClientId, SessPid}, []). %% @doc Set session stats --spec(set_session_stats(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - emqx_stats:stats()) -> true). +-spec(set_session_stats(emqx_types:client_id(), emqx_stats:stats()) -> true). set_session_stats(ClientId, Stats) when is_binary(ClientId) -> - set_session_stats({ClientId, self()}, Stats); -set_session_stats(Session = {ClientId, SPid}, Stats) when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_STATS_TAB, {Session, Stats}). + set_session_stats(ClientId, self(), Stats). -%% @doc Lookup a session from registry --spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). -lookup_session(ClientId) -> +-spec(set_session_stats(emqx_types:client_id(), pid(), emqx_stats:stats()) -> true). +set_session_stats(ClientId, SessPid, Stats) when is_binary(ClientId), is_pid(SessPid) -> + ets:insert(?SESSION_STATS_TAB, {{ClientId, SessPid}, Stats}). + +%% @doc Lookup session pid. +-spec(lookup_session_pids(emqx_types:client_id()) -> list(pid())). +lookup_session_pids(ClientId) -> case emqx_sm_registry:is_enabled() of true -> emqx_sm_registry:lookup_session(ClientId); - false -> ets:lookup(?SESSION_TAB, ClientId) + false -> ets:lookup(?SESSION_TAB, ClientId, []) end. %% @doc Dispatch a message to the session. -spec(dispatch(emqx_types:client_id(), emqx_topic:topic(), emqx_types:message()) -> any()). dispatch(ClientId, Topic, Msg) -> - case lookup_session_pid(ClientId) of - Pid when is_pid(Pid) -> - Pid ! {dispatch, Topic, Msg}; - undefined -> + case lookup_session_pids(ClientId) of + [SessPid|_] when is_pid(SessPid) -> + SessPid ! {dispatch, Topic, Msg}; + [] -> emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg]) end. -%% @doc Lookup session pid. --spec(lookup_session_pid(emqx_types:client_id()) -> pid() | undefined). -lookup_session_pid(ClientId) -> - emqx_tables:lookup_value(?SESSION_TAB, ClientId). - notify(Event) -> gen_server:cast(?SM, {notify, Event}). @@ -203,43 +227,36 @@ init([]) -> ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), - ok = emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), + ok = emqx_stats:update_interval(sess_stats, fun ?MODULE:stats_fun/0), {ok, #{sess_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[SM] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({notify, {registered, ClientId, SPid}}, State = #{sess_pmon := PMon}) -> - {noreply, State#{sess_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; +handle_cast({notify, {registered, ClientId, SessPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:monitor(SessPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{sess_pmon := PMon}) -> - {noreply, State#{sess_pmon := emqx_pmon:demonitor(SPid, PMon)}}; +handle_cast({notify, {unregistered, SessPid}}, State = #{sess_pmon := PMon}) -> + {noreply, State#{sess_pmon := emqx_pmon:demonitor(SessPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{sess_pmon := PMon}) -> - case emqx_pmon:find(DownPid, PMon) of - undefined -> - {noreply, State}; - ClientId -> - Session = {ClientId, DownPid}, - case ets:member(?SESSION_ATTRS_TAB, Session) of - true -> - ok = emqx_pool:async_submit(fun do_unregister_session/1, [Session]); - false -> ok - end, - {noreply, State#{sess_pmon := emqx_pmon:erase(DownPid, PMon)}} - end; +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{sess_pmon := PMon}) -> + SessPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(SessPids, PMon), + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{sess_pmon := PMon1}}; handle_info(Info, State) -> emqx_logger:error("[SM] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(sm_stats). + emqx_stats:cancel_update(sess_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -248,6 +265,15 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ +clean_down({SessPid, ClientId}) -> + Session = {ClientId, SessPid}, + case ets:member(?SESSION_TAB, ClientId) + orelse ets:member(?SESSION_ATTRS_TAB, Session) of + true -> + do_unregister_session(Session); + false -> ok + end. + stats_fun() -> safe_update_stats(?SESSION_TAB, 'sessions/count', 'sessions/max'), safe_update_stats(?SESSION_P_TAB, 'sessions/persistent/count', 'sessions/persistent/max'). diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl index 1d0df61bf..a7f44d771 100644 --- a/src/emqx_sm_registry.erl +++ b/src/emqx_sm_registry.erl @@ -43,10 +43,9 @@ start_link() -> is_enabled() -> emqx_config:get_env(enable_session_registry, true). --spec(lookup_session(emqx_types:client_id()) - -> list({emqx_types:client_id(), session_pid()})). +-spec(lookup_session(emqx_types:client_id()) -> list(session_pid())). lookup_session(ClientId) -> - [{ClientId, SessPid} || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)]. + [SessPid || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)]. -spec(register_session({emqx_types:client_id(), session_pid()}) -> ok). register_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> diff --git a/test/emqx_banned_SUITE.erl b/test/emqx_banned_SUITE.erl index 9d4c85134..7e7434e61 100644 --- a/test/emqx_banned_SUITE.erl +++ b/test/emqx_banned_SUITE.erl @@ -18,9 +18,7 @@ -compile(nowarn_export_all). -include("emqx.hrl"). - -include("emqx_mqtt.hrl"). - -include_lib("eunit/include/eunit.hrl"). all() -> [t_banned_all]. @@ -29,18 +27,27 @@ t_banned_all(_) -> emqx_ct_broker_helpers:run_setup_steps(), emqx_banned:start_link(), TimeNow = erlang:system_time(second), - Banned = #banned{who = {client_id, <<"TestClient">>}, + Banned = #banned{who = {client_id, <<"TestClient">>}, reason = <<"test">>, - by = <<"banned suite">>, - desc = <<"test">>, - until = TimeNow + 1}, + by = <<"banned suite">>, + desc = <<"test">>, + until = TimeNow + 1}, ok = emqx_banned:add(Banned), % here is not expire banned test because its check interval is greater than 5 mins, but its effect has been confirmed - ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), timer:sleep(2500), - ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), ok = emqx_banned:add(Banned), - ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), - emqx_banned:del({client_id, <<"TestClient">>}), - ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + emqx_banned:delete({client_id, <<"TestClient">>}), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), emqx_ct_broker_helpers:run_teardown_steps(). + diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index 5e29e075e..b720849f6 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -24,14 +24,16 @@ all() -> [t_register_unregister_connection]. t_register_unregister_connection(_) -> {ok, _} = emqx_cm_sup:start_link(), Pid = self(), - emqx_cm:register_connection(<<"conn1">>), - emqx_cm:register_connection({<<"conn2">>, Pid}, [{port, 8080}, {ip, "192.168.0.1"}]), + ok = emqx_cm:register_connection(<<"conn1">>), + ok emqx_cm:register_connection(<<"conn2">>, Pid), + true = emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}]), + true = emqx_cm:set_conn_attrs(<<"conn2">>, Pid, [{port, 8080}, {ip, "192.168.0.1"}]), timer:sleep(2000), - [{<<"conn1">>, Pid}] = emqx_cm:lookup_connection(<<"conn1">>), - [{<<"conn2">>, Pid}] = emqx_cm:lookup_connection(<<"conn2">>), - Pid = emqx_cm:lookup_conn_pid(<<"conn1">>), - emqx_cm:unregister_connection(<<"conn1">>), - [] = emqx_cm:lookup_connection(<<"conn1">>), - [{port, 8080}, {ip, "192.168.0.1"}] = emqx_cm:get_conn_attrs({<<"conn2">>, Pid}), - emqx_cm:set_conn_stats(<<"conn2">>, [[{count, 1}, {max, 2}]]), - [[{count, 1}, {max, 2}]] = emqx_cm:get_conn_stats({<<"conn2">>, Pid}). + ?assertEqual(Pid, emqx_cm:lookup_conn_pid(<<"conn1">>)), + ?assertEqual(Pid, emqx_cm:lookup_conn_pid(<<"conn2">>)), + ok = emqx_cm:unregister_connection(<<"conn1">>), + ?assertEqual(undefined, emqx_cm:lookup_conn_pid(<<"conn1">>)), + ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs({<<"conn2">>, Pid})), + true = emqx_cm:set_conn_stats(<<"conn2">>, [{count, 1}, {max, 2}]), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats({<<"conn2">>, Pid})). + diff --git a/test/emqx_pmon_SUITE.erl b/test/emqx_pmon_SUITE.erl new file mode 100644 index 000000000..67e8cf4d7 --- /dev/null +++ b/test/emqx_pmon_SUITE.erl @@ -0,0 +1,48 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pmon_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_monitor, t_find, t_erase]. + +t_monitor(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + PMon2 = emqx_pmon:demonitor(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)). + +t_find(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + ?assertEqual({ok, val}, emqx_pmon:find(self(), PMon1)), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(error, emqx_pmon:find(self(), PMon2)). + +t_erase(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)), + {Items, PMon3} = emqx_pmon:erase_all([self()], PMon1), + ?assertEqual([{self(), val}], Items), + ?assertEqual(0, emqx_pmon:count(PMon3)). + diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl index 1ac0ea308..ab408b8e0 100644 --- a/test/emqx_sequence_SUITE.erl +++ b/test/emqx_sequence_SUITE.erl @@ -34,5 +34,5 @@ sequence_generate(_) -> ?assertEqual(0, reclaim(seqtab, key)), ?assertEqual(false, ets:member(seqtab, key)), ?assertEqual(1, nextval(seqtab, key)), - ?assert(emqx_sequence:delete(seqtab). + ?assert(emqx_sequence:delete(seqtab)). diff --git a/test/emqx_sm_SUITE.erl b/test/emqx_sm_SUITE.erl index 3aed3090e..008d4b6e6 100644 --- a/test/emqx_sm_SUITE.erl +++ b/test/emqx_sm_SUITE.erl @@ -34,16 +34,14 @@ t_open_close_session(_) -> topic_alias_maximum => 0, will_msg => undefined}, {ok, SPid} = emqx_sm:open_session(Attrs), - [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>), - SPid = emqx_sm:lookup_session_pid(<<"client">>), + ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)), {ok, NewConnPid} = emqx_mock_client:start_link(<<"client">>), {ok, SPid, true} = emqx_sm:open_session(Attrs#{clean_start => false, conn_pid => NewConnPid}), - [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>), - SAttrs = emqx_sm:get_session_attrs({<<"client">>, SPid}), - <<"client">> = proplists:get_value(client_id, SAttrs), - Session = {<<"client">>, SPid}, - emqx_sm:set_session_stats(Session, {open, true}), - {open, true} = emqx_sm:get_session_stats(Session), + ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)), + SAttrs = emqx_sm:get_session_attrs(<<"client">>, SPid), + ?assertEqual(<<"client">>, proplists:get_value(client_id, SAttrs)), + emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}]), + ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)), ok = emqx_sm:close_session(SPid), - [] = emqx_sm:lookup_session(<<"client">>), + ?assertEqual([], emqx_sm:lookup_session_pids(<<"client">>)), emqx_ct_broker_helpers:run_teardown_steps(). From dc06c0beab77f30a6063dc966d6d1ee924913db1 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Tue, 18 Dec 2018 15:11:04 +0800 Subject: [PATCH 36/54] Remove 'topic_alias_maximum' from session's state --- src/emqx_protocol.erl | 53 +++++++++--------------- src/emqx_session.erl | 95 +++++++++++++++++++------------------------ 2 files changed, 62 insertions(+), 86 deletions(-) diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 573b913f7..5751f78fe 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -619,14 +619,12 @@ maybe_use_username_as_clientid(ClientId, undefined, _PState) -> ClientId; maybe_use_username_as_clientid(ClientId, Username, #pstate{zone = Zone}) -> case emqx_zone:get_env(Zone, use_username_as_clientid, false) of - true -> - Username; - false -> - ClientId + true -> Username; + false -> ClientId end. %%------------------------------------------------------------------------------ -%% Assign a clientid +%% Assign a clientId maybe_assign_client_id(PState = #pstate{client_id = <<>>, ack_props = AckProps}) -> ClientId = emqx_guid:to_base62(emqx_guid:gen()), @@ -650,41 +648,30 @@ try_open_session(PState = #pstate{zone = Zone, clean_start => CleanStart, will_msg => WillMsg }, - SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}, {topic_alias_maximum, PState}]), + SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}]), case emqx_sm:open_session(SessAttrs1) of {ok, SPid} -> {ok, SPid, false}; Other -> Other end. -set_session_attrs({max_inflight, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) -> - maps:put(max_inflight, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Receive-Maximum', ConnProps, 65535); - true -> - emqx_zone:get_env(Zone, max_inflight, 65535) - end, SessAttrs); -set_session_attrs({expiry_interval, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps, clean_start = CleanStart}}, SessAttrs) -> - maps:put(expiry_interval, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Session-Expiry-Interval', ConnProps, 0); - true -> - case CleanStart of - true -> 0; - false -> - emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff) - end - end, SessAttrs); -set_session_attrs({topic_alias_maximum, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) -> - maps:put(topic_alias_maximum, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Topic-Alias-Maximum', ConnProps, 0); - true -> - emqx_zone:get_env(Zone, max_topic_alias, 0) - end, SessAttrs); -set_session_attrs({_, #pstate{}}, SessAttrs) -> - SessAttrs. +set_session_attrs({max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(max_inflight, get_property('Receive-Maximum', ConnProps, 65535), SessAttrs); +set_session_attrs({max_inflight, #pstate{zone = Zone}}, SessAttrs) -> + maps:put(max_inflight, emqx_zone:get_env(Zone, max_inflight, 65535), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(expiry_interval, get_property('Session-Expiry-Interval', ConnProps, 0), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{zone = Zone, clean_start = CleanStart}}, SessAttrs) -> + maps:put(expiry_interval, case CleanStart of + true -> 0; + false -> emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff) + end, SessAttrs); + +set_session_attrs(_, SessAttrs) -> + SessAttrs. authenticate(Credentials, Password) -> case emqx_access_control:authenticate(Credentials, Password) of diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 12718b9fc..fde76f0b6 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -147,8 +147,6 @@ %% Created at created_at :: erlang:timestamp(), - topic_alias_maximum :: pos_integer(), - will_msg :: emqx:message(), will_delay_timer :: reference() | undefined @@ -341,7 +339,6 @@ init([Parent, #{zone := Zone, clean_start := CleanStart, expiry_interval := ExpiryInterval, max_inflight := MaxInflight, - topic_alias_maximum := TopicAliasMaximum, will_msg := WillMsg}]) -> emqx_logger:set_metadata_client_id(ClientId), process_flag(trap_exit, true), @@ -367,7 +364,6 @@ init([Parent, #{zone := Zone, deliver_stats = 0, enqueue_stats = 0, created_at = os:timestamp(), - topic_alias_maximum = TopicAliasMaximum, will_msg = WillMsg }, emqx_sm:register_session(ClientId, attrs(State)), @@ -518,22 +514,23 @@ handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight end; %% RESUME: -handle_cast({resume, #{conn_pid := ConnPid, - will_msg := WillMsg, - expiry_interval := SessionExpiryInterval, - max_inflight := MaxInflight, - topic_alias_maximum := TopicAliasMaximum}}, State = #state{client_id = ClientId, - conn_pid = OldConnPid, - clean_start = CleanStart, - retry_timer = RetryTimer, - await_rel_timer = AwaitTimer, - expiry_timer = ExpireTimer, - will_delay_timer = WillDelayTimer}) -> +handle_cast({resume, #{conn_pid := ConnPid, + will_msg := WillMsg, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight}}, + State = #state{client_id = ClientId, + conn_pid = OldConnPid, + clean_start = CleanStart, + retry_timer = RetryTimer, + await_rel_timer = AwaitTimer, + expiry_timer = ExpireTimer, + will_delay_timer = WillDelayTimer}) -> ?LOG(info, "Resumed by connection ~p ", [ConnPid], State), %% Cancel Timers - lists:foreach(fun emqx_misc:cancel_timer/1, [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]), + lists:foreach(fun emqx_misc:cancel_timer/1, + [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]), case kick(ClientId, OldConnPid, ConnPid) of ok -> ?LOG(warning, "Connection ~p kickout ~p", [ConnPid, OldConnPid], State); @@ -542,19 +539,18 @@ handle_cast({resume, #{conn_pid := ConnPid, true = link(ConnPid), - State1 = State#state{conn_pid = ConnPid, - binding = binding(ConnPid), - old_conn_pid = OldConnPid, - clean_start = false, - retry_timer = undefined, - awaiting_rel = #{}, - await_rel_timer = undefined, - expiry_timer = undefined, - expiry_interval = SessionExpiryInterval, - inflight = emqx_inflight:update_size(MaxInflight, State#state.inflight), - topic_alias_maximum = TopicAliasMaximum, - will_delay_timer = undefined, - will_msg = WillMsg}, + State1 = State#state{conn_pid = ConnPid, + binding = binding(ConnPid), + old_conn_pid = OldConnPid, + clean_start = false, + retry_timer = undefined, + awaiting_rel = #{}, + await_rel_timer = undefined, + expiry_timer = undefined, + expiry_interval = ExpiryInterval, + inflight = emqx_inflight:update_size(MaxInflight, State#state.inflight), + will_delay_timer = undefined, + will_msg = WillMsg}, %% Clean Session: true -> false??? CleanStart andalso emqx_sm:set_session_attrs(ClientId, attrs(State1)), @@ -573,9 +569,10 @@ handle_cast(Msg, State) -> %% Batch dispatch handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) -> - {noreply, lists:foldl(fun(Msg, NewState) -> - element(2, handle_info({dispatch, Topic, Msg}, NewState)) - end, State, Msgs)}; + noreply(lists:foldl( + fun(Msg, St) -> + element(2, handle_info({dispatch, Topic, Msg}, St)) + end, State, Msgs)); %% Dispatch message handle_info({dispatch, Topic, Msg = #message{}}, State) -> @@ -584,12 +581,11 @@ handle_info({dispatch, Topic, Msg = #message{}}, State) -> %% Require ack, but we do not have connection %% negative ack the message so it can try the next subscriber in the group ok = emqx_shared_sub:nack_no_connection(Msg), - noreply(State); + {noreply, State}; false -> - handle_dispatch(Topic, Msg, State) + noreply(handle_dispatch(Topic, Msg, State)) end; - %% Do nothing if the client has been disconnected. handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) -> noreply(State#state{retry_timer = undefined}); @@ -663,25 +659,17 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -has_connection(#state{conn_pid = Pid}) -> is_pid(Pid) andalso is_process_alive(Pid). +has_connection(#state{conn_pid = Pid}) -> + is_pid(Pid) andalso is_process_alive(Pid). -handle_dispatch(Topic, Msg = #message{headers = Headers}, - State = #state{subscriptions = SubMap, - topic_alias_maximum = TopicAliasMaximum - }) -> - TopicAlias = maps:get('Topic-Alias', Headers, undefined), - if - TopicAlias =:= undefined orelse TopicAlias =< TopicAliasMaximum -> - noreply(case maps:find(Topic, SubMap) of - {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); - {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); - error -> - dispatch(emqx_message:unset_flag(dup, Msg), State) - end); - true -> - noreply(State) +handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) -> + case maps:find(Topic, SubMap) of + {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); + {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); + error -> + dispatch(emqx_message:unset_flag(dup, Msg), State) end. suback(_From, undefined, _ReasonCodes) -> @@ -1011,3 +999,4 @@ noreply(State) -> shutdown(Reason, State) -> {stop, {shutdown, Reason}, State}. + From 1e2c5db36cc666df6da158e879a2cb215002626a Mon Sep 17 00:00:00 2001 From: turtled Date: Tue, 18 Dec 2018 10:21:00 +0800 Subject: [PATCH 37/54] Modify batch size --- src/emqx_broker_helper.erl | 2 +- src/emqx_cm.erl | 2 +- src/emqx_sm.erl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index c8e2b5eab..1ea3f3668 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -31,7 +31,7 @@ -define(SUBSEQ, emqx_subseq). -define(SHARD, 1024). --define(BATCH_SIZE, 10000). +-define(BATCH_SIZE, 100000). -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 62cb417de..b8a57bc50 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -42,7 +42,7 @@ -define(CONN_ATTRS_TAB, emqx_conn_attrs). -define(CONN_STATS_TAB, emqx_conn_stats). --define(BATCH_SIZE, 10000). +-define(BATCH_SIZE, 100000). %% @doc Start the connection manager. -spec(start_link() -> emqx_types:startlink_ret()). diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 281893fb0..97a1b689c 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -49,7 +49,7 @@ -define(SESSION_ATTRS_TAB, emqx_session_attrs). -define(SESSION_STATS_TAB, emqx_session_stats). --define(BATCH_SIZE, 10000). +-define(BATCH_SIZE, 100000). -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> @@ -201,7 +201,7 @@ set_session_stats(ClientId, SessPid, Stats) when is_binary(ClientId), is_pid(Ses lookup_session_pids(ClientId) -> case emqx_sm_registry:is_enabled() of true -> emqx_sm_registry:lookup_session(ClientId); - false -> ets:lookup(?SESSION_TAB, ClientId, []) + false -> emqx_tables:lookup_value(?SESSION_TAB, ClientId, []) end. %% @doc Dispatch a message to the session. From c7fa4b1b15acec3d805f94ad8612260a9d1d7234 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 18 Dec 2018 16:50:47 +0800 Subject: [PATCH 38/54] Revert vm args in emqx conf b (#2070) * Revert changes in emqx.conf for backward compatibility --- etc/emqx.conf | 93 ++++++++++++++++++++++++++++++++++++++ etc/vm.args | 12 ++--- priv/emqx.schema | 114 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 6 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 7d4d8c966..4144593d8 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -160,6 +160,99 @@ node.name = emqx@127.0.0.1 ## Value: String node.cookie = emqxsecretcookie +## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable +## heartbeat, or set the value as 'on' +## +## Value: on +## +## vm.args: -heart +## node.heartbeat = on + +## Sets the number of threads in async thread pool. Valid range is 0-1024. +## +## See: http://erlang.org/doc/man/erl.html +## +## Value: 0-1024 +## +## vm.args: +A Number +node.async_threads = 32 + +## Sets the maximum number of simultaneously existing processes for this +## system if a Number is passed as value. +## +## See: http://erlang.org/doc/man/erl.html +## +## Value: Number [1024-134217727] +## +## vm.args: +P Number +node.process_limit = 2048000 + +## Sets the maximum number of simultaneously existing ports for this system. +## +## See: http://erlang.org/doc/man/erl.html +## +## Value: Number [1024-134217727] +## +## vm.args: +Q Number +node.max_ports = 1024000 + +## Set the distribution buffer busy limit (dist_buf_busy_limit). +## +## See: http://erlang.org/doc/man/erl.html +## +## Value: Number [1KB-2GB] +## +## vm.args: +zdbbl size +node.dist_buffer_size = 8MB + +## Sets the maximum number of ETS tables. Note that mnesia and SSL will +## create temporary ETS tables. +## +## Value: Number +## +## vm.args: +e Number +node.max_ets_tables = 256000 + +## Tweak GC to run more often. +## +## Value: Number [0-65535] +## +## vm.args: -env ERL_FULLSWEEP_AFTER Number +node.fullsweep_after = 1000 + +## Crash dump log file. +## +## Value: Log file +node.crash_dump = {{ platform_log_dir }}/crash.dump + +## Specify the erlang distributed protocol. +## +## Value: Enum +## - inet_tcp: the default; handles TCP streams with IPv4 addressing. +## - inet6_tcp: handles TCP with IPv6 addressing. +## - inet_tls: using TLS for Erlang Distribution. +## +## vm.args: -proto_dist inet_tcp +node.proto_dist = inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## +## Value: File +## +## vm.args: -ssl_dist_optfile +## node.ssl_dist_optfile = {{ platform_etc_dir }}/ssl_dist.conf + +## Sets the net_kernel tick time. TickTime is specified in seconds. +## Notice that all communicating nodes are to have the same TickTime +## value specified. +## +## See: http://www.erlang.org/doc/man/kernel_app.html#net_ticktime +## +## Value: Number +## +## vm.args: -kernel net_ticktime Number +node.dist_net_ticktime = 60 + ## Sets the port range for the listener socket of a distributed Erlang node. ## Note that if there are firewalls between clustered nodes, this port segment ## for nodes’ communication should be allowed. diff --git a/etc/vm.args b/etc/vm.args index d0992f470..43e6467d9 100644 --- a/etc/vm.args +++ b/etc/vm.args @@ -10,19 +10,19 @@ ## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. ## Sets the maximum number of simultaneously existing processes for this system. -+P 256000 +#+P 2048000 ## Sets the maximum number of simultaneously existing ports for this system. -+Q 262144 +#+Q 1024000 ## Sets the maximum number of ETS tables -+e 256000 +#+e 256000 ## Sets the maximum number of atoms the virtual machine can handle. #+t 1048576 ## Set the location of crash dumps --env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump +#-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump ## Set how many times generational garbages collections can be done without ## forcing a fullsweep collection. @@ -46,13 +46,13 @@ #-kernel net_ticktime 60 ## Sets the distribution buffer busy limit (dist_buf_busy_limit). -+zdbbl 8192 +#+zdbbl 8192 ## Sets default scheduler hint for port parallelism. +spp true ## Sets the number of threads in async thread pool. Valid range is 0-1024. -+A 8 +#+A 8 ## Sets the default heap size of processes to the size Size. #+hms 233 diff --git a/priv/emqx.schema b/priv/emqx.schema index 7c37383c4..7ecc66a35 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -173,11 +173,125 @@ end}. {default, "emqx@127.0.0.1"} ]}. +%% @doc The erlang distributed protocol +{mapping, "node.proto_dist", "vm_args.-proto_dist", [ + %{default, "inet_tcp"}, + {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, + hidden +]}. + +%% @doc Specify SSL Options in the file if using SSL for erlang distribution +{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ + {datatype, string}, + hidden +]}. + %% @doc Secret cookie for distributed erlang node {mapping, "node.cookie", "vm_args.-setcookie", [ {default, "emqxsecretcookie"} ]}. +%% @doc http://erlang.org/doc/man/heart.html +{mapping, "node.heartbeat", "vm_args.-heart", [ + {datatype, flag}, + hidden +]}. + +{translation, "vm_args.-heart", fun(Conf) -> + case cuttlefish:conf_get("node.heartbeat", Conf) of + true -> ""; + false -> cuttlefish:invalid("should be 'on' or comment the line!") + end +end}. + +%% @doc More information at: http://erlang.org/doc/man/erl.html +{mapping, "node.async_threads", "vm_args.+A", [ + {default, 64}, + {datatype, integer}, + {validators, ["range:0-1024"]} +]}. + +%% @doc Erlang Process Limit +{mapping, "node.process_limit", "vm_args.+P", [ + {datatype, integer}, + {default, 256000}, + hidden +]}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_PORTS, R16+ uses +Q +%% @doc The number of concurrent ports/sockets +%% Valid range is 1024-134217727 +{mapping, "node.max_ports", + cuttlefish:otp("R16", "vm_args.+Q", "vm_args.-env ERL_MAX_PORTS"), [ + {default, 262144}, + {datatype, integer}, + {validators, ["range4ports"]} +]}. + +{validator, "range4ports", "must be 1024 to 134217727", + fun(X) -> X >= 1024 andalso X =< 134217727 end}. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ + {datatype, bytesize}, + {commented, "32MB"}, + hidden, + {validators, ["zdbbl_range"]} +]}. + +{translation, "vm_args.+zdbbl", + fun(Conf) -> + ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), + case ZDBBL of + undefined -> undefined; + X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end + end +}. + +{validator, "zdbbl_range", "must be between 1KB and 2097151KB", + fun(ZDBBL) -> + %% 2097151KB = 2147482624 + ZDBBL >= 1024 andalso ZDBBL =< 2147482624 + end +}. + +%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 +{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ + {default, 1000}, + {datatype, integer}, + hidden, + {validators, ["positive_integer"]} +]}. + +{validator, "positive_integer", "must be a positive integer", + fun(X) -> X >= 0 end}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, +%% R16+ uses +e +%% @doc The ETS table limit +{mapping, "node.max_ets_tables", + cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ + {default, 256000}, + {datatype, integer}, + hidden +]}. + +%% @doc Set the location of crash dumps +{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ + {default, "{{crash_dump}}"}, + {datatype, file}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime +{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ + {commented, 60}, + {datatype, integer}, + hidden +]}. + %% @doc http://www.erlang.org/doc/man/kernel_app.html {mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ {commented, 6369}, From 892d9439b9144ce88b2187f7ab02c4e8cb7cf0c4 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Wed, 19 Dec 2018 16:49:35 +0800 Subject: [PATCH 39/54] Implement a new session supervisor. (#2077) --- src/emqx_session_sup.erl | 256 +++++++++++++++++++++++++++++++++++++++ src/emqx_sm.erl | 39 ++---- src/emqx_sm_sup.erl | 47 ++++--- 3 files changed, 297 insertions(+), 45 deletions(-) create mode 100644 src/emqx_session_sup.erl diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl new file mode 100644 index 000000000..efa64815b --- /dev/null +++ b/src/emqx_session_sup.erl @@ -0,0 +1,256 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_session_sup). + +-behaviour(gen_server). + +-export([start_link/1]). +-export([start_session/1, count_sessions/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-type(shutdown() :: brutal_kill | infinity | pos_integer()). + +-record(state, { + sessions :: #{pid() => emqx_types:client_id()}, + mfargs :: mfa(), + shutdown :: shutdown(), + clean_down :: fun() + }). + +-define(SUP, ?MODULE). +-define(BATCH_EXIT, 100000). +-define(ERROR_MSG(Format, Args), + error_logger:error_msg("[~s] " ++ Format, [?MODULE | Args])). + +%% @doc Start session supervisor. +-spec(start_link(map()) -> emqx_types:startlink_ret()). +start_link(SessSpec) when is_map(SessSpec) -> + gen_server:start_link({local, ?SUP}, ?MODULE, [SessSpec], []). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Start a session. +-spec(start_session(map()) -> emqx_types:startlink_ret()). +start_session(SessAttrs) -> + gen_server:call(?SUP, {start_session, SessAttrs}, infinity). + +%% @doc Count sessions. +-spec(count_sessions() -> non_neg_integer()). +count_sessions() -> + gen_server:call(?SUP, count_sessions, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Spec]) -> + process_flag(trap_exit, true), + MFA = maps:get(start, Spec), + Shutdown = maps:get(shutdown, Spec, brutal_kill), + CleanDown = maps:get(clean_down, Spec, undefined), + State = #state{sessions = #{}, + mfargs = MFA, + shutdown = Shutdown, + clean_down = CleanDown + }, + {ok, State}. + +handle_call({start_session, SessAttrs = #{client_id := ClientId}}, _From, + State = #state{sessions = SessMap, mfargs = {M, F, Args}}) -> + try erlang:apply(M, F, [SessAttrs | Args]) of + {ok, Pid} -> + reply({ok, Pid}, State#state{sessions = maps:put(Pid, ClientId, SessMap)}); + ignore -> + reply(ignore, State); + {error, Reason} -> + reply({error, Reason}, State) + catch + _:Error:Stk -> + ?ERROR_MSG("Failed to start session ~p: ~p, stacktrace:~n~p", + [ClientId, Error, Stk]), + reply({error, Error}, State) + end; + +handle_call(count_sessions, _From, State = #state{sessions = SessMap}) -> + {reply, maps:size(SessMap), State}; + +handle_call(Req, _From, State) -> + ?ERROR_MSG("unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?ERROR_MSG("unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'EXIT', Pid, _Reason}, State = #state{sessions = SessMap, clean_down = CleanDown}) -> + SessPids = [Pid | drain_exit(?BATCH_EXIT, [])], + {SessItems, SessMap1} = erase_all(SessPids, SessMap), + (CleanDown =:= undefined) + orelse emqx_pool:async_submit( + fun lists:foreach/2, [CleanDown, SessItems]), + {noreply, State#state{sessions = SessMap1}}; + +handle_info(Info, State) -> + ?ERROR_MSG("unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, State) -> + terminate_children(State). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +drain_exit(0, Acc) -> + lists:reverse(Acc); +drain_exit(Cnt, Acc) -> + receive + {'EXIT', Pid, _Reason} -> + drain_exit(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + +erase_all(Pids, Map) -> + lists:foldl( + fun(Pid, {Acc, M}) -> + case maps:take(Pid, M) of + {Val, M1} -> + {[{Val, Pid}|Acc], M1}; + error -> + {Acc, M} + end + end, {[], Map}, Pids). + +terminate_children(State = #state{sessions = SessMap, shutdown = Shutdown}) -> + {Pids, EStack0} = monitor_children(SessMap), + Sz = sets:size(Pids), + EStack = + case Shutdown of + brutal_kill -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + infinity -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + Time when is_integer(Time) -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + TRef = erlang:start_timer(Time, self(), kill), + wait_children(Shutdown, Pids, Sz, TRef, EStack0) + end, + %% Unroll stacked errors and report them + dict:fold(fun(Reason, Pid, _) -> + report_error(connection_shutdown_error, Reason, Pid, State) + end, ok, EStack). + +monitor_children(SessMap) -> + lists:foldl( + fun(Pid, {Pids, EStack}) -> + case monitor_child(Pid) of + ok -> + {sets:add_element(Pid, Pids), EStack}; + {error, normal} -> + {Pids, EStack}; + {error, Reason} -> + {Pids, dict:append(Reason, Pid, EStack)} + end + end, {sets:new(), dict:new()}, maps:keys(SessMap)). + +%% Help function to shutdown/2 switches from link to monitor approach +monitor_child(Pid) -> + %% Do the monitor operation first so that if the child dies + %% before the monitoring is done causing a 'DOWN'-message with + %% reason noproc, we will get the real reason in the 'EXIT'-message + %% unless a naughty child has already done unlink... + erlang:monitor(process, Pid), + unlink(Pid), + + receive + %% If the child dies before the unlik we must empty + %% the mail-box of the 'EXIT'-message and the 'DOWN'-message. + {'EXIT', Pid, Reason} -> + receive + {'DOWN', _, process, Pid, _} -> + {error, Reason} + end + after 0 -> + %% If a naughty child did unlink and the child dies before + %% monitor the result will be that shutdown/2 receives a + %% 'DOWN'-message with reason noproc. + %% If the child should die after the unlink there + %% will be a 'DOWN'-message with a correct reason + %% that will be handled in shutdown/2. + ok + end. + +wait_children(_Shutdown, _Pids, 0, undefined, EStack) -> + EStack; +wait_children(_Shutdown, _Pids, 0, TRef, EStack) -> + %% If the timer has expired before its cancellation, we must empty the + %% mail-box of the 'timeout'-message. + erlang:cancel_timer(TRef), + receive + {timeout, TRef, kill} -> + EStack + after 0 -> + EStack + end; + +%%TODO: Copied from supervisor.erl, rewrite it later. +wait_children(brutal_kill, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, killed} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), + Sz-1, TRef, dict:append(Reason, Pid, EStack)) + end; + +wait_children(Shutdown, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, shutdown} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, normal} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, + TRef, dict:append(Reason, Pid, EStack)); + {timeout, TRef, kill} -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz-1, undefined, EStack) + end. + +report_error(Error, Reason, Pid, #state{mfargs = MFA}) -> + SupName = list_to_atom("esockd_connection_sup - " ++ pid_to_list(self())), + ErrorMsg = [{supervisor, SupName}, + {errorContext, Error}, + {reason, Reason}, + {offender, [{pid, Pid}, + {name, connection}, + {mfargs, MFA}]}], + error_logger:error_report(supervisor_report, ErrorMsg). + +reply(Repy, State) -> + {reply, Repy, State}. + diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 97a1b689c..ab270f1af 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -37,6 +37,9 @@ %% Internal function for stats -export([stats_fun/0]). +%% Internal function for emqx_session_sup +-export([clean_down/1]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -60,7 +63,7 @@ start_link() -> open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid := ConnPid}) -> CleanStart = fun(_) -> ok = discard_session(ClientId, ConnPid), - emqx_session:start_link(SessAttrs) + emqx_session_sup:start_session(SessAttrs) end, emqx_sm_locker:trans(ClientId, CleanStart); @@ -70,7 +73,7 @@ open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> {ok, SessPid} -> {ok, SessPid, true}; {error, not_found} -> - emqx_session:start_link(SessAttrs) + emqx_session_sup:start_session(SessAttrs) end end, emqx_sm_locker:trans(ClientId, ResumeStart). @@ -130,8 +133,7 @@ register_session(ClientId) when is_binary(ClientId) -> register_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> Session = {ClientId, SessPid}, true = ets:insert(?SESSION_TAB, Session), - ok = emqx_sm_registry:register_session(Session), - notify({registered, ClientId, SessPid}). + emqx_sm_registry:register_session(Session). %% @doc Unregister a session -spec(unregister_session(emqx_types:client_id()) -> ok). @@ -140,11 +142,7 @@ unregister_session(ClientId) when is_binary(ClientId) -> -spec(unregister_session(emqx_types:client_id(), pid()) -> ok). unregister_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> - ok = do_unregister_session({ClientId, SessPid}), - notify({unregistered, SessPid}). - -%% @private -do_unregister_session(Session) -> + Session = {ClientId, SessPid}, true = ets:delete(?SESSION_STATS_TAB, Session), true = ets:delete(?SESSION_ATTRS_TAB, Session), true = ets:delete_object(?SESSION_P_TAB, Session), @@ -214,9 +212,6 @@ dispatch(ClientId, Topic, Msg) -> emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg]) end. -notify(Event) -> - gen_server:cast(?SM, {notify, Event}). - %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ @@ -228,29 +223,16 @@ init([]) -> ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), ok = emqx_stats:update_interval(sess_stats, fun ?MODULE:stats_fun/0), - {ok, #{sess_pmon => emqx_pmon:new()}}. + {ok, #{}}. handle_call(Req, _From, State) -> emqx_logger:error("[SM] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({notify, {registered, ClientId, SessPid}}, State = #{sess_pmon := PMon}) -> - {noreply, State#{sess_pmon := emqx_pmon:monitor(SessPid, ClientId, PMon)}}; - -handle_cast({notify, {unregistered, SessPid}}, State = #{sess_pmon := PMon}) -> - {noreply, State#{sess_pmon := emqx_pmon:demonitor(SessPid, PMon)}}; - handle_cast(Msg, State) -> emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{sess_pmon := PMon}) -> - SessPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], - {Items, PMon1} = emqx_pmon:erase_all(SessPids, PMon), - ok = emqx_pool:async_submit( - fun lists:foreach/2, [fun clean_down/1, Items]), - {noreply, State#{sess_pmon := PMon1}}; - handle_info(Info, State) -> emqx_logger:error("[SM] unexpected info: ~p", [Info]), {noreply, State}. @@ -265,12 +247,11 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -clean_down({SessPid, ClientId}) -> - Session = {ClientId, SessPid}, +clean_down(Session = {ClientId, SessPid}) -> case ets:member(?SESSION_TAB, ClientId) orelse ets:member(?SESSION_ATTRS_TAB, Session) of true -> - do_unregister_session(Session); + unregister_session(ClientId, SessPid); false -> ok end. diff --git a/src/emqx_sm_sup.erl b/src/emqx_sm_sup.erl index 0be9facb0..317cd11db 100644 --- a/src/emqx_sm_sup.erl +++ b/src/emqx_sm_sup.erl @@ -26,25 +26,40 @@ start_link() -> init([]) -> %% Session locker - Locker = #{id => locker, - start => {emqx_sm_locker, start_link, []}, - restart => permanent, + Locker = #{id => locker, + start => {emqx_sm_locker, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm_locker]}, + type => worker, + modules => [emqx_sm_locker] + }, %% Session registry - Registry = #{id => registry, - start => {emqx_sm_registry, start_link, []}, - restart => permanent, + Registry = #{id => registry, + start => {emqx_sm_registry, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm_registry]}, + type => worker, + modules => [emqx_sm_registry] + }, %% Session Manager - Manager = #{id => manager, - start => {emqx_sm, start_link, []}, - restart => permanent, + Manager = #{id => manager, + start => {emqx_sm, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm]}, - {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager]}}. + type => worker, + modules => [emqx_sm] + }, + %% Session Sup + SessSpec = #{start => {emqx_session, start_link, []}, + shutdown => brutal_kill, + clean_down => fun emqx_sm:clean_down/1 + }, + SessionSup = #{id => session_sup, + start => {emqx_session_sup, start_link, [SessSpec ]}, + restart => transient, + shutdown => infinity, + type => supervisor, + modules => [emqx_session_sup] + }, + {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager, SessionSup]}}. From 7a1ec580b0716b53dec54bb6bf3c864fa8294692 Mon Sep 17 00:00:00 2001 From: turtled Date: Wed, 19 Dec 2018 17:14:06 +0800 Subject: [PATCH 40/54] Update broker test cases --- src/emqx_broker.erl | 2 +- src/emqx_session.erl | 2 +- test/emqx_broker_SUITE.erl | 63 ++++++++++++++----------------------- test/emqx_session_SUITE.erl | 4 +-- 4 files changed, 27 insertions(+), 44 deletions(-) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 6d1ee98cb..429c6097d 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -87,7 +87,7 @@ subscribe(Topic) when is_binary(Topic) -> -spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - subscribe(Topic, SubId, #{}); + subscribe(Topic, SubId, #{qos => 0}); subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 1e186b0ab..9117dd1b8 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -482,7 +482,7 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, lists:foldr(fun({Topic, _SubOpts}, {Acc, SubMap}) -> case maps:find(Topic, SubMap) of {ok, SubOpts} -> - ok = emqx_broker:unsubscribe(Topic, ClientId), + ok = emqx_broker:unsubscribe(Topic), emqx_hooks:run('session.unsubscribed', [#{client_id => ClientId}, Topic, SubOpts]), {[?RC_SUCCESS|Acc], maps:remove(Topic, SubMap)}; error -> diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index f9a6dcf40..b2031a61c 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -36,7 +36,6 @@ groups() -> [ {pubsub, [sequence], [subscribe_unsubscribe, publish, pubsub, - t_local_subscribe, t_shared_subscribe, dispatch_with_no_sub, 'pubsub#', 'pubsub+']}, @@ -61,14 +60,14 @@ subscribe_unsubscribe(_) -> ok = emqx:subscribe(<<"topic">>, <<"clientId">>), ok = emqx:subscribe(<<"topic/1">>, <<"clientId">>, #{ qos => 1 }), ok = emqx:subscribe(<<"topic/2">>, <<"clientId">>, #{ qos => 2 }), - true = emqx:subscribed(<<"topic">>, <<"clientId">>), + true = emqx:subscribed(<<"clientId">>, <<"topic">>), Topics = emqx:topics(), lists:foreach(fun(Topic) -> ?assert(lists:member(Topic, Topics)) end, Topics), - ok = emqx:unsubscribe(<<"topic">>, <<"clientId">>), - ok = emqx:unsubscribe(<<"topic/1">>, <<"clientId">>), - ok = emqx:unsubscribe(<<"topic/2">>, <<"clientId">>). + ok = emqx:unsubscribe(<<"topic">>), + ok = emqx:unsubscribe(<<"topic/1">>), + ok = emqx:unsubscribe(<<"topic/2">>). publish(_) -> Msg = emqx_message:make(ct, <<"test/pubsub">>, <<"hello">>), @@ -85,18 +84,25 @@ dispatch_with_no_sub(_) -> pubsub(_) -> true = emqx:is_running(node()), Self = self(), - Subscriber = {Self, <<"clientId">>}, - ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 1 }), - #{qos := 1} = ets:lookup_element(emqx_suboption, {<<"a/b/c">>, Subscriber}, 2), - #{qos := 1} = emqx:get_subopts(<<"a/b/c">>, Subscriber), - true = emqx:set_subopts(<<"a/b/c">>, Subscriber, #{qos => 0}), - #{qos := 0} = emqx:get_subopts(<<"a/b/c">>, Subscriber), - ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 2 }), + Subscriber = <<"clientId">>, + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 1 }), + #{qos := 1} = ets:lookup_element(emqx_suboption, {Self, <<"a/b/c">>}, 2), + #{qos := 1} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + true = emqx_broker:set_subopts(<<"a/b/c">>, #{qos => 0}), + #{qos := 0} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 2 }), %% ct:log("Emq Sub: ~p.~n", [ets:lookup(emqx_suboption, {<<"a/b/c">>, Subscriber})]), timer:sleep(10), - [{Self, <<"clientId">>}] = emqx_broker:subscribers(<<"a/b/c">>), + [Self] = emqx_broker:subscribers(<<"a/b/c">>), emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)), - ?assert(receive {dispatch, <<"a/b/c">>, _ } -> true; P -> ct:log("Receive Message: ~p~n",[P]) after 2 -> false end), + ?assert( + receive {dispatch, <<"a/b/c">>, _ } -> + true; + P -> + ct:log("Receive Message: ~p~n",[P]) + after 2 -> + false + end), spawn(fun() -> emqx:subscribe(<<"a/b/c">>), emqx:subscribe(<<"c/d/e">>), @@ -106,38 +112,15 @@ pubsub(_) -> timer:sleep(20), emqx:unsubscribe(<<"a/b/c">>). -t_local_subscribe(_) -> - ok = emqx:subscribe(<<"$local/topic0">>), - ok = emqx:subscribe(<<"$local/topic1">>, <<"clientId">>), - ok = emqx:subscribe(<<"$local/topic2">>, <<"clientId">>, #{ qos => 2 }), - timer:sleep(10), - ?assertEqual([{self(), undefined}], emqx:subscribers("$local/topic0")), - ?assertEqual([{self(), <<"clientId">>}], emqx:subscribers("$local/topic1")), - ?assertEqual([{<<"$local/topic1">>, #{ qos => 0 }}, - {<<"$local/topic2">>, #{ qos => 2 }}], - emqx:subscriptions({self(), <<"clientId">>})), - ?assertEqual(ok, emqx:unsubscribe("$local/topic0")), - ?assertEqual(ok, emqx:unsubscribe("$local/topic0")), - ?assertEqual(ok, emqx:unsubscribe("$local/topic1", <<"clientId">>)), - ?assertEqual(ok, emqx:unsubscribe("$local/topic2", <<"clientId">>)), - ?assertEqual([], emqx:subscribers("topic1")), - ?assertEqual([], emqx:subscriptions({self(), <<"clientId">>})). - t_shared_subscribe(_) -> - emqx:subscribe("$local/$share/group1/topic1"), emqx:subscribe("$share/group2/topic2"), emqx:subscribe("$queue/topic3"), timer:sleep(10), - ct:log("share subscriptions: ~p~n", [emqx:subscriptions({self(), undefined})]), - ?assertEqual([{self(), undefined}], emqx:subscribers(<<"$local/$share/group1/topic1">>)), - ?assertEqual([{<<"$local/$share/group1/topic1">>, #{qos => 0}}, - {<<"$queue/topic3">>, #{qos => 0}}, - {<<"$share/group2/topic2">>, #{qos => 0}}], - lists:sort(emqx:subscriptions({self(), undefined}))), - emqx:unsubscribe("$local/$share/group1/topic1"), + ct:log("share subscriptions: ~p~n", [emqx:subscriptions(self())]), + ?assertEqual(2, length(emqx:subscriptions(self()))), emqx:unsubscribe("$share/group2/topic2"), emqx:unsubscribe("$queue/topic3"), - ?assertEqual([], lists:sort(emqx:subscriptions(self()))). + ?assertEqual(0, length(emqx:subscriptions(self()))). 'pubsub#'(_) -> emqx:subscribe(<<"a/#">>), diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index a04a0b82b..2e597340d 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -53,7 +53,7 @@ t_session_all(_) -> emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 2}}]), emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 1}}]), timer:sleep(200), - [{<<"topic">>, _}] = emqx:subscriptions({SPid, <<"ClientId">>}), + [{<<"topic">>, _}] = emqx:subscriptions(SPid), emqx_session:publish(SPid, 1, Message1), timer:sleep(200), {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid), @@ -76,5 +76,5 @@ t_session_all(_) -> 1 = proplists:get_value(subscriptions_count, Stats), emqx_session:unsubscribe(SPid, [<<"topic">>]), timer:sleep(200), - [] = emqx:subscriptions({SPid, <<"clientId">>}), + [] = emqx:subscriptions(SPid), emqx_mock_client:close_session(ConnPid). From 42fc8f5811d551aebee9578ed16da5ad90eae043 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 10:17:14 +0800 Subject: [PATCH 41/54] Improve the session module --- src/emqx_session.erl | 170 ++++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 82 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 9117dd1b8..9c28884c9 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -158,8 +158,6 @@ -export_type([attr/0]). --define(TIMEOUT, 60000). - -define(LOG(Level, Format, Args, _State), emqx_logger:Level("[Session] " ++ Format, Args)). @@ -261,9 +259,11 @@ subscribe(SPid, PacketId, Properties, TopicFilters) -> publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) -> %% Publish QoS0 message directly emqx_broker:publish(Msg); + publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) -> %% Publish QoS1 message directly emqx_broker:publish(Msg); + publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> %% Register QoS2 message packet ID (and timestamp) to session, then publish case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of @@ -275,6 +275,7 @@ publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> puback(SPid, PacketId) -> gen_server:cast(SPid, {puback, PacketId, ?RC_SUCCESS}). +-spec(puback(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) -> ok). puback(SPid, PacketId, ReasonCode) -> gen_server:cast(SPid, {puback, PacketId, ReasonCode}). @@ -322,7 +323,7 @@ discard(SPid, ByPid) -> -spec(update_expiry_interval(spid(), timeout()) -> ok). update_expiry_interval(SPid, Interval) -> - gen_server:cast(SPid, {expiry_interval, Interval}). + gen_server:cast(SPid, {update_expiry_interval, Interval}). -spec(close(spid()) -> ok). close(SPid) -> @@ -332,39 +333,39 @@ close(SPid) -> %% gen_server callbacks %%------------------------------------------------------------------------------ -init([Parent, #{zone := Zone, - client_id := ClientId, - username := Username, - conn_pid := ConnPid, - clean_start := CleanStart, - expiry_interval := ExpiryInterval, - max_inflight := MaxInflight, - will_msg := WillMsg}]) -> - emqx_logger:set_metadata_client_id(ClientId), +init([Parent, #{zone := Zone, + client_id := ClientId, + username := Username, + conn_pid := ConnPid, + clean_start := CleanStart, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight, + will_msg := WillMsg}]) -> process_flag(trap_exit, true), true = link(ConnPid), + emqx_logger:set_metadata_client_id(ClientId), IdleTimout = get_env(Zone, idle_timeout, 30000), - State = #state{idle_timeout = IdleTimout, - clean_start = CleanStart, - binding = binding(ConnPid), - client_id = ClientId, - username = Username, - conn_pid = ConnPid, - subscriptions = #{}, - max_subscriptions = get_env(Zone, max_subscriptions, 0), - upgrade_qos = get_env(Zone, upgrade_qos, false), - inflight = emqx_inflight:new(MaxInflight), - mqueue = init_mqueue(Zone), - retry_interval = get_env(Zone, retry_interval, 0), - awaiting_rel = #{}, - await_rel_timeout = get_env(Zone, await_rel_timeout), - max_awaiting_rel = get_env(Zone, max_awaiting_rel), - expiry_interval = ExpiryInterval, - enable_stats = get_env(Zone, enable_stats, true), - deliver_stats = 0, - enqueue_stats = 0, - created_at = os:timestamp(), - will_msg = WillMsg + State = #state{idle_timeout = IdleTimout, + clean_start = CleanStart, + binding = binding(ConnPid), + client_id = ClientId, + username = Username, + conn_pid = ConnPid, + subscriptions = #{}, + max_subscriptions = get_env(Zone, max_subscriptions, 0), + upgrade_qos = get_env(Zone, upgrade_qos, false), + inflight = emqx_inflight:new(MaxInflight), + mqueue = init_mqueue(Zone), + retry_interval = get_env(Zone, retry_interval, 0), + awaiting_rel = #{}, + await_rel_timeout = get_env(Zone, await_rel_timeout), + max_awaiting_rel = get_env(Zone, max_awaiting_rel), + expiry_interval = ExpiryInterval, + enable_stats = get_env(Zone, enable_stats, true), + deliver_stats = 0, + enqueue_stats = 0, + created_at = os:timestamp(), + will_msg = WillMsg }, ok = emqx_sm:register_session(ClientId, self()), true = emqx_sm:set_session_attrs(ClientId, attrs(State)), @@ -397,53 +398,56 @@ handle_call(stats, _From, State) -> handle_call({discard, ByPid}, _From, State = #state{conn_pid = undefined}) -> ?LOG(warning, "Discarded by ~p", [ByPid], State), - {stop, {shutdown, discard}, ok, State}; + {stop, discarded, ok, State}; handle_call({discard, ByPid}, _From, State = #state{client_id = ClientId, conn_pid = ConnPid}) -> ?LOG(warning, "Conn ~p is discarded by ~p", [ConnPid, ByPid], State), ConnPid ! {shutdown, discard, {ClientId, ByPid}}, - {stop, {shutdown, discard}, ok, State}; + {stop, discarded, ok, State}; %% PUBLISH: This is only to register packetId to session state. %% The actual message dispatching should be done by the caller (e.g. connection) process. handle_call({register_publish_packet_id, PacketId, Ts}, _From, State = #state{awaiting_rel = AwaitingRel}) -> - reply(case is_awaiting_full(State) of - false -> - case maps:is_key(PacketId, AwaitingRel) of - true -> - {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; - false -> - State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, - {ok, ensure_await_rel_timer(State1)} - end; - true -> - emqx_metrics:trans(inc, 'messages/qos2/dropped'), - ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), - {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State} - end); + reply( + case is_awaiting_full(State) of + false -> + case maps:is_key(PacketId, AwaitingRel) of + true -> + {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; + false -> + State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, + {ok, ensure_await_rel_timer(State1)} + end; + true -> + ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), + emqx_metrics:trans(inc, 'messages/qos2/dropped'), + {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State} + end); %% PUBREC: handle_call({pubrec, PacketId, _ReasonCode}, _From, State = #state{inflight = Inflight}) -> - reply(case emqx_inflight:contain(PacketId, Inflight) of - true -> - {ok, acked(pubrec, PacketId, State)}; - false -> - emqx_metrics:trans(inc, 'packets/pubrec/missed'), - ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), - {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} - end); + reply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + {ok, acked(pubrec, PacketId, State)}; + false -> + ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrec/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); %% PUBREL: handle_call({pubrel, PacketId, _ReasonCode}, _From, State = #state{awaiting_rel = AwaitingRel}) -> - reply(case maps:take(PacketId, AwaitingRel) of - {_Ts, AwaitingRel1} -> - {ok, State#state{awaiting_rel = AwaitingRel1}}; - error -> - emqx_metrics:trans(inc, 'packets/pubrel/missed'), - ?LOG(warning, "Cannot find PUBREL: ~w", [PacketId], State), - {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} - end); + reply( + case maps:take(PacketId, AwaitingRel) of + {_Ts, AwaitingRel1} -> + {ok, State#state{awaiting_rel = AwaitingRel1}}; + error -> + ?LOG(warning, "The PUBREL PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrel/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); handle_call(close, _From, State) -> {stop, normal, ok, State}; @@ -494,25 +498,27 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, %% PUBACK: handle_cast({puback, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> - case emqx_inflight:contain(PacketId, Inflight) of - true -> - noreply(dequeue(acked(puback, PacketId, State))); - false -> - ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), - emqx_metrics:trans(inc, 'packets/puback/missed'), - {noreply, State} - end; + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + dequeue(acked(puback, PacketId, State)); + false -> + ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/puback/missed'), + State + end); %% PUBCOMP: handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> - case emqx_inflight:contain(PacketId, Inflight) of - true -> - noreply(dequeue(acked(pubcomp, PacketId, State))); - false -> - ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), - emqx_metrics:trans(inc, 'packets/pubcomp/missed'), - {noreply, State} - end; + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + dequeue(acked(pubcomp, PacketId, State)); + false -> + ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubcomp/missed'), + State + end); %% RESUME: handle_cast({resume, #{conn_pid := ConnPid, @@ -561,7 +567,7 @@ handle_cast({resume, #{conn_pid := ConnPid, %% Replay delivery and Dequeue pending messages noreply(dequeue(retry_delivery(true, State1))); -handle_cast({expiry_interval, Interval}, State) -> +handle_cast({update_expiry_interval, Interval}, State) -> {noreply, State#state{expiry_interval = Interval}}; handle_cast(Msg, State) -> From d8276042136f9b2e453863bdba90dad7172e6e17 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 10:56:56 +0800 Subject: [PATCH 42/54] Remove the ensure_stats_timer/1 call from reply/2 and noreply/1 --- src/emqx_session.erl | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 9c28884c9..e9bc0ff98 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -417,7 +417,7 @@ handle_call({register_publish_packet_id, PacketId, Ts}, _From, {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; false -> State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, - {ok, ensure_await_rel_timer(State1)} + {ok, ensure_stats_timer(ensure_await_rel_timer(State1))} end; true -> ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), @@ -430,7 +430,7 @@ handle_call({pubrec, PacketId, _ReasonCode}, _From, State = #state{inflight = In reply( case emqx_inflight:contain(PacketId, Inflight) of true -> - {ok, acked(pubrec, PacketId, State)}; + {ok, ensure_stats_timer(acked(pubrec, PacketId, State))}; false -> ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), emqx_metrics:trans(inc, 'packets/pubrec/missed'), @@ -442,7 +442,7 @@ handle_call({pubrel, PacketId, _ReasonCode}, _From, State = #state{awaiting_rel reply( case maps:take(PacketId, AwaitingRel) of {_Ts, AwaitingRel1} -> - {ok, State#state{awaiting_rel = AwaitingRel1}}; + {ok, ensure_stats_timer(State#state{awaiting_rel = AwaitingRel1})}; error -> ?LOG(warning, "The PUBREL PacketId ~w is not found", [PacketId], State), emqx_metrics:trans(inc, 'packets/pubrel/missed'), @@ -477,7 +477,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, end} end, {[], Subscriptions}, TopicFilters), suback(FromPid, PacketId, ReasonCodes), - noreply(State#state{subscriptions = Subscriptions1}); + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); %% UNSUBSCRIBE: handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, @@ -494,14 +494,14 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, end end, {[], Subscriptions}, TopicFilters), unsuback(From, PacketId, ReasonCodes), - noreply(State#state{subscriptions = Subscriptions1}); + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); %% PUBACK: handle_cast({puback, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> noreply( case emqx_inflight:contain(PacketId, Inflight) of true -> - dequeue(acked(puback, PacketId, State)); + ensure_stats_timer(dequeue(acked(puback, PacketId, State))); false -> ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), emqx_metrics:trans(inc, 'packets/puback/missed'), @@ -513,7 +513,7 @@ handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight noreply( case emqx_inflight:contain(PacketId, Inflight) of true -> - dequeue(acked(pubcomp, PacketId, State)); + ensure_stats_timer(dequeue(acked(pubcomp, PacketId, State))); false -> ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), emqx_metrics:trans(inc, 'packets/pubcomp/missed'), @@ -565,7 +565,7 @@ handle_cast({resume, #{conn_pid := ConnPid, emqx_hooks:run('session.resumed', [#{client_id => ClientId}, attrs(State)]), %% Replay delivery and Dequeue pending messages - noreply(dequeue(retry_delivery(true, State1))); + noreply(ensure_stats_timer(dequeue(retry_delivery(true, State1)))); handle_cast({update_expiry_interval, Interval}, State) -> {noreply, State#state{expiry_interval = Interval}}; @@ -590,7 +590,7 @@ handle_info({dispatch, Topic, Msg = #message{}}, State) -> ok = emqx_shared_sub:nack_no_connection(Msg), {noreply, State}; false -> - noreply(handle_dispatch(Topic, Msg, State)) + noreply(ensure_stats_timer(handle_dispatch(Topic, Msg, State))) end; %% Do nothing if the client has been disconnected. @@ -601,11 +601,11 @@ handle_info({timeout, Timer, retry_delivery}, State = #state{retry_timer = Timer noreply(retry_delivery(false, State#state{retry_timer = undefined})); handle_info({timeout, Timer, check_awaiting_rel}, State = #state{await_rel_timer = Timer}) -> - noreply(expire_awaiting_rel(State#state{await_rel_timer = undefined})); + State1 = State#state{await_rel_timer = undefined}, + noreply(ensure_stats_timer(expire_awaiting_rel(State1))); handle_info({timeout, Timer, emit_stats}, - State = #state{client_id = ClientId, - stats_timer = Timer}) -> + State = #state{client_id = ClientId, stats_timer = Timer}) -> emqx_metrics:commit(), _ = emqx_sm:set_session_stats(ClientId, stats(State)), NewState = State#state{stats_timer = undefined}, @@ -931,8 +931,7 @@ dequeue(State = #state{inflight = Inflight}) -> dequeue2(State = #state{mqueue = Q}) -> case emqx_mqueue:out(Q) of - {empty, _Q} -> - State; + {empty, _Q} -> State; {{value, Msg}, Q1} -> %% Dequeue more dequeue(dispatch(Msg, State#state{mqueue = Q1})) @@ -972,7 +971,8 @@ ensure_will_delay_timer(State = #state{will_msg = #message{headers = #{'Will-Del ensure_will_delay_timer(State) -> State. -ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined, +ensure_stats_timer(State = #state{enable_stats = true, + stats_timer = undefined, idle_timeout = IdleTimeout}) -> State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; ensure_stats_timer(State) -> @@ -1010,11 +1010,11 @@ reply({Reply, State}) -> reply(Reply, State). reply(Reply, State) -> - {reply, Reply, ensure_stats_timer(State)}. + {reply, Reply, State}. noreply(State) -> - {noreply, ensure_stats_timer(State)}. + {noreply, State}. shutdown(Reason, State) -> - {stop, {shutdown, Reason}, State}. + {stop, Reason, State}. From 6e1b47f1f9cd364eb9a4cda6400e856baa1005ca Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 14:17:16 +0800 Subject: [PATCH 43/54] Improve the emqx_connection module Rename 'publish_limit' to 'pub_limit' 'try ... of ... catch' to replace 'case catch' --- src/emqx_connection.erl | 75 +++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index e1d4011fc..2f596068c 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -16,15 +16,12 @@ -behaviour(gen_server). --define(LOG_HEADER, "[TCP]"). - -include("emqx.hrl"). -include("emqx_mqtt.hrl"). -include("logger.hrl"). -export([start_link/3]). --export([info/1, attrs/1]). --export([stats/1]). +-export([info/1, attrs/1, stats/1]). -export([kick/1]). -export([session/1]). @@ -46,11 +43,12 @@ stats_timer, incoming, rate_limit, - publish_limit, + pub_limit, limit_timer, idle_timeout }). +-define(LOG_HEADER, "[TCP]"). -define(DEFAULT_ACTIVE_N, 100). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). @@ -65,22 +63,22 @@ start_link(Transport, Socket, Options) -> info(CPid) when is_pid(CPid) -> call(CPid, info); -info(#state{transport = Transport, - socket = Socket, - peername = Peername, - sockname = Sockname, - conn_state = ConnState, - active_n = ActiveN, - rate_limit = RateLimit, - publish_limit = PubLimit, - proto_state = ProtoState}) -> +info(#state{transport = Transport, + socket = Socket, + peername = Peername, + sockname = Sockname, + conn_state = ConnState, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState}) -> ConnInfo = [{socktype, Transport:type(Socket)}, {peername, Peername}, {sockname, Sockname}, {conn_state, ConnState}, {active_n, ActiveN}, {rate_limit, esockd_rate_limit:info(RateLimit)}, - {publish_limit, esockd_rate_limit:info(PubLimit)}], + {pub_limit, esockd_rate_limit:info(PubLimit)}], ProtoInfo = emqx_protocol:info(ProtoState), lists:usort(lists:append(ConnInfo, ProtoInfo)). @@ -139,22 +137,21 @@ init([Transport, RawSocket, Options]) -> peercert => Peercert, sendfun => SendFun}, Options), ParserState = emqx_protocol:parser(ProtoState), - State = run_socket(#state{transport = Transport, - socket = Socket, - peername = Peername, - conn_state = running, - active_n = ActiveN, - rate_limit = RateLimit, - publish_limit = PubLimit, - proto_state = ProtoState, - parser_state = ParserState, - enable_stats = EnableStats, - idle_timeout = IdleTimout + State = run_socket(#state{transport = Transport, + socket = Socket, + peername = Peername, + conn_state = running, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState, + parser_state = ParserState, + enable_stats = EnableStats, + idle_timeout = IdleTimout }), GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), ok = emqx_gc:init(GcPolicy), ok = emqx_misc:init_proc_mng_policy(Zone), - emqx_logger:set_metadata_peername(esockd_net:format(Peername)), gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State, self(), IdleTimout); @@ -213,6 +210,7 @@ handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> {error, Reason} -> shutdown(Reason, State) end; + handle_info({timeout, Timer, emit_stats}, State = #state{stats_timer = Timer, proto_state = ProtoState @@ -231,6 +229,7 @@ handle_info({timeout, Timer, emit_stats}, ?LOG(warning, "shutdown due to ~p", [Reason]), shutdown(Reason, NewState) end; + handle_info(timeout, State) -> shutdown(idle_timeout, State); @@ -331,9 +330,9 @@ handle_packet(<<>>, State) -> handle_packet(Data, State = #state{proto_state = ProtoState, parser_state = ParserState, idle_timeout = IdleTimeout}) -> - case catch emqx_frame:parse(Data, ParserState) of - {more, NewParserState} -> - {noreply, State#state{parser_state = NewParserState}, IdleTimeout}; + try emqx_frame:parse(Data, ParserState) of + {more, ParserState1} -> + {noreply, State#state{parser_state = ParserState1}, IdleTimeout}; {ok, Packet = ?PACKET(Type), Rest} -> emqx_metrics:received(Packet), case emqx_protocol:received(Packet, ProtoState) of @@ -348,11 +347,12 @@ handle_packet(Data, State = #state{proto_state = ProtoState, {stop, Error, ProtoState1} -> stop(Error, State#state{proto_state = ProtoState1}) end; - {error, Error} -> - ?LOG(error, "Framing error - ~p", [Error]), - shutdown(Error, State); - {'EXIT', Reason} -> - ?LOG(error, "Parse failed for ~p~nError data:~p", [Reason, Data]), + {error, Reason} -> + ?LOG(error, "Parse frame error - ~p", [Reason]), + shutdown(Reason, State) + catch + _:Error -> + ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]), shutdown(parse_error, State) end. @@ -370,9 +370,9 @@ inc_publish_cnt(_Type, State) -> %% Ensure rate limit %%------------------------------------------------------------------------------ -ensure_rate_limit(State = #state{rate_limit = Rl, publish_limit = Pl, +ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl, incoming = #{packets := Packets, bytes := Bytes}}) -> - ensure_rate_limit([{Pl, #state.publish_limit, Packets}, + ensure_rate_limit([{Pl, #state.pub_limit, Packets}, {Rl, #state.rate_limit, Bytes}], State). ensure_rate_limit([], State) -> @@ -421,3 +421,4 @@ maybe_gc(#state{}, {publish, _PacketId, #message{payload = Payload}}) -> ok = emqx_gc:inc(1, Oct); maybe_gc(_, _) -> ok. + From 14a12e0c6c791a5576f1e037f03afb5fe5b8efa3 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 14:37:30 +0800 Subject: [PATCH 44/54] Move the 'LOG_HEADER' macro above '-include(logger.hrl)' --- src/emqx_connection.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 2f596068c..79c244a11 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -18,6 +18,8 @@ -include("emqx.hrl"). -include("emqx_mqtt.hrl"). + +-define(LOG_HEADER, "[MQTT]"). -include("logger.hrl"). -export([start_link/3]). @@ -48,7 +50,6 @@ idle_timeout }). --define(LOG_HEADER, "[TCP]"). -define(DEFAULT_ACTIVE_N, 100). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). From 14cffcf7fbb7b3dae227554476152efd4a339b27 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 16:45:25 +0800 Subject: [PATCH 45/54] Add the 'emqx_pd' module Add utility functions for erlang process dictionary Add test cases for emqx_pd --- Makefile | 2 +- src/emqx_connection.erl | 31 +++++++++++++------------------ src/emqx_pd.erl | 33 +++++++++++++++++++++++++++++++++ test/emqx_cm_SUITE.erl | 3 ++- test/emqx_pd_SUITE.erl | 31 +++++++++++++++++++++++++++++++ test/emqx_sm_SUITE.erl | 2 ++ 6 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 src/emqx_pd.erl create mode 100644 test/emqx_pd_SUITE.erl diff --git a/Makefile b/Makefile index 68d383940..374b74fb7 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ - emqx_hooks emqx_batch emqx_sequence emqx_pmon + emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 79c244a11..2cf647beb 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -249,12 +249,13 @@ handle_info({tcp, _Sock, Data}, State) -> ?LOG(debug, "RECV ~p", [Data]), Size = iolist_size(Data), emqx_metrics:trans(inc, 'bytes/received', Size), + emqx_pd:update_counter(incoming_bytes, Size), Incoming = #{bytes => Size, packets => 0}, handle_packet(Data, State#state{incoming = Incoming}); %% Rate limit here, cool:) handle_info({tcp_passive, _Sock}, State) -> - {noreply, ensure_rate_limit(State)}; + {noreply, run_socket(ensure_rate_limit(State))}; handle_info({tcp_error, _Sock, Reason}, State) -> shutdown(Reason, State); @@ -336,10 +337,10 @@ handle_packet(Data, State = #state{proto_state = ProtoState, {noreply, State#state{parser_state = ParserState1}, IdleTimeout}; {ok, Packet = ?PACKET(Type), Rest} -> emqx_metrics:received(Packet), + (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1), case emqx_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - NewState = State#state{proto_state = ProtoState1}, - handle_packet(Rest, inc_publish_cnt(Type, reset_parser(NewState))); + handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1})); {error, Reason} -> ?LOG(error, "Process packet error - ~p", [Reason]), shutdown(Reason, State); @@ -360,28 +361,21 @@ handle_packet(Data, State = #state{proto_state = ProtoState, reset_parser(State = #state{proto_state = ProtoState}) -> State#state{parser_state = emqx_protocol:parser(ProtoState)}. -inc_publish_cnt(Type, State = #state{incoming = Incoming = #{packets := Cnt}}) - when Type == ?PUBLISH; Type == ?SUBSCRIBE -> - State#state{incoming = Incoming#{packets := Cnt + 1}}; - -inc_publish_cnt(_Type, State) -> - State. - %%------------------------------------------------------------------------------ %% Ensure rate limit %%------------------------------------------------------------------------------ -ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl, - incoming = #{packets := Packets, bytes := Bytes}}) -> - ensure_rate_limit([{Pl, #state.pub_limit, Packets}, - {Rl, #state.rate_limit, Bytes}], State). +ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl}) -> + Limiters = [{Pl, #state.pub_limit, emqx_pd:reset_counter(incoming_pubs)}, + {Rl, #state.rate_limit, emqx_pd:reset_counter(incoming_bytes)}], + ensure_rate_limit(Limiters, State). ensure_rate_limit([], State) -> - run_socket(State); -ensure_rate_limit([{undefined, _Pos, _Num}|Limiters], State) -> + State; +ensure_rate_limit([{undefined, _Pos, _Cnt}|Limiters], State) -> ensure_rate_limit(Limiters, State); -ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) -> - case esockd_rate_limit:check(Num, Rl) of +ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) -> + case esockd_rate_limit:check(Cnt, Rl) of {0, Rl1} -> ensure_rate_limit(Limiters, setelement(Pos, State, Rl1)); {Pause, Rl1} -> @@ -423,3 +417,4 @@ maybe_gc(#state{}, {publish, _PacketId, #message{payload = Payload}}) -> maybe_gc(_, _) -> ok. + diff --git a/src/emqx_pd.erl b/src/emqx_pd.erl new file mode 100644 index 000000000..ce1e7723c --- /dev/null +++ b/src/emqx_pd.erl @@ -0,0 +1,33 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc The utility functions for erlang process dictionary. +-module(emqx_pd). + +-export([update_counter/2, get_counter/1, reset_counter/1]). + +-type(key() :: term()). + +-spec(update_counter(key(), number()) -> undefined | number()). +update_counter(Key, Inc) -> + put(Key, get_counter(Key) + Inc). + +-spec(get_counter(key()) -> number()). +get_counter(Key) -> + case get(Key) of undefined -> 0; Cnt -> Cnt end. + +-spec(reset_counter(key()) -> number()). +reset_counter(Key) -> + case put(Key, 0) of undefined -> 0; Cnt -> Cnt end. + diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index b720849f6..208294474 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -18,6 +18,7 @@ -compile(nowarn_export_all). -include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). all() -> [t_register_unregister_connection]. @@ -25,7 +26,7 @@ t_register_unregister_connection(_) -> {ok, _} = emqx_cm_sup:start_link(), Pid = self(), ok = emqx_cm:register_connection(<<"conn1">>), - ok emqx_cm:register_connection(<<"conn2">>, Pid), + ok = emqx_cm:register_connection(<<"conn2">>, Pid), true = emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}]), true = emqx_cm:set_conn_attrs(<<"conn2">>, Pid, [{port, 8080}, {ip, "192.168.0.1"}]), timer:sleep(2000), diff --git a/test/emqx_pd_SUITE.erl b/test/emqx_pd_SUITE.erl new file mode 100644 index 000000000..e53fa7539 --- /dev/null +++ b/test/emqx_pd_SUITE.erl @@ -0,0 +1,31 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pd_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> [update_counter]. + +update_counter(_) -> + ?assertEqual(undefined, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(1, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(2, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(3, emqx_pd:get_counter(bytes)), + ?assertEqual(3, emqx_pd:reset_counter(bytes)), + ?assertEqual(0, emqx_pd:get_counter(bytes)). + diff --git a/test/emqx_sm_SUITE.erl b/test/emqx_sm_SUITE.erl index 008d4b6e6..6adb7c388 100644 --- a/test/emqx_sm_SUITE.erl +++ b/test/emqx_sm_SUITE.erl @@ -19,6 +19,8 @@ -compile(export_all). -compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). + all() -> [t_open_close_session]. t_open_close_session(_) -> From 6b538d2363afac57acc148cf07238c74cbb697ac Mon Sep 17 00:00:00 2001 From: YoukiLin <1045735402@qq.com> Date: Thu, 20 Dec 2018 21:03:40 +0800 Subject: [PATCH 46/54] Add test cases for 'emqx_cm' and 'emqx_sm' (#2073) Add test cases for 'emqx_cm' and 'emqx_cn' --- src/emqx_ctl.erl | 10 ---- test/emqx_client_SUITE.erl | 8 ++-- test/emqx_cm_SUITE.erl | 64 +++++++++++++++++++------- test/emqx_sm_SUITE.erl | 93 ++++++++++++++++++++++++++++---------- 4 files changed, 121 insertions(+), 54 deletions(-) diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl index c00556eb7..1d2fb13a3 100644 --- a/src/emqx_ctl.erl +++ b/src/emqx_ctl.erl @@ -54,15 +54,6 @@ run_command([Cmd | Args]) -> run_command(list_to_atom(Cmd), Args). -spec(run_command(cmd(), [string()]) -> ok | {error, term()}). -% run_command(set, []) -> -% emqx_mgmt_cli_cfg:set_usage(), ok; - -% run_command(set, Args) -> -% emqx_mgmt_cli_cfg:run(["config" | Args]), ok; - -% run_command(show, Args) -> -% emqx_mgmt_cli_cfg:run(["config" | Args]), ok; - run_command(help, []) -> usage(); run_command(Cmd, Args) when is_atom(Cmd) -> @@ -160,4 +151,3 @@ register_command_test_() -> }. -endif. - diff --git a/test/emqx_client_SUITE.erl b/test/emqx_client_SUITE.erl index 021109606..13303bfdc 100644 --- a/test/emqx_client_SUITE.erl +++ b/test/emqx_client_SUITE.erl @@ -32,9 +32,8 @@ <<"+/+">>, <<"TopicA/#">>]). all() -> - [ {group, mqttv4}, - {group, mqttv5} - ]. + [{group, mqttv4}, + {group, mqttv5}]. groups() -> [{mqttv4, [non_parallel_tests], @@ -48,8 +47,7 @@ groups() -> dollar_topics_test]}, {mqttv5, [non_parallel_tests], [request_response, - share_sub_request_topic]} -]. + share_sub_request_topic]}]. init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index b720849f6..08a773d40 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -1,3 +1,4 @@ + %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,23 +18,54 @@ -compile(export_all). -compile(nowarn_export_all). +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -include("emqx_mqtt.hrl"). -all() -> [t_register_unregister_connection]. +all() -> [{group, cm}]. -t_register_unregister_connection(_) -> - {ok, _} = emqx_cm_sup:start_link(), - Pid = self(), - ok = emqx_cm:register_connection(<<"conn1">>), - ok emqx_cm:register_connection(<<"conn2">>, Pid), - true = emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}]), - true = emqx_cm:set_conn_attrs(<<"conn2">>, Pid, [{port, 8080}, {ip, "192.168.0.1"}]), - timer:sleep(2000), - ?assertEqual(Pid, emqx_cm:lookup_conn_pid(<<"conn1">>)), - ?assertEqual(Pid, emqx_cm:lookup_conn_pid(<<"conn2">>)), - ok = emqx_cm:unregister_connection(<<"conn1">>), - ?assertEqual(undefined, emqx_cm:lookup_conn_pid(<<"conn1">>)), - ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs({<<"conn2">>, Pid})), - true = emqx_cm:set_conn_stats(<<"conn2">>, [{count, 1}, {max, 2}]), - ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats({<<"conn2">>, Pid})). +groups() -> + [{cm, [non_parallel_tests], + [t_get_set_conn_attrs, + t_get_set_conn_stats, + t_lookup_conn_pid]}]. +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +init_per_testcase(_TestCase, Config) -> + register_connection(), + Config. + +end_per_testcase(_TestCase, _Config) -> + unregister_connection(), + ok. + +t_get_set_conn_attrs(_) -> + ?assert(emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}])), + ?assert(emqx_cm:set_conn_attrs(<<"conn2">>, self(), [{port, 8080}, {ip, "192.168.0.2"}])), + ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs(<<"conn1">>)), + ?assertEqual([{port, 8080}, {ip, "192.168.0.2"}], emqx_cm:get_conn_attrs(<<"conn2">>, self())). + +t_get_set_conn_stats(_) -> + ?assert(emqx_cm:set_conn_stats(<<"conn1">>, [{count, 1}, {max, 2}])), + ?assert(emqx_cm:set_conn_stats(<<"conn2">>, self(), [{count, 1}, {max, 2}])), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn1">>)), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn2">>, self())). + +t_lookup_conn_pid(_) -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>, self())), + ?assertEqual(self(), emqx_cm:lookup_conn_pid(<<"conn1">>)). + +register_connection() -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:register_connection(<<"conn2">>, self())). + +unregister_connection() -> + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn2">>, self())). diff --git a/test/emqx_sm_SUITE.erl b/test/emqx_sm_SUITE.erl index 008d4b6e6..407e2c92b 100644 --- a/test/emqx_sm_SUITE.erl +++ b/test/emqx_sm_SUITE.erl @@ -14,34 +14,81 @@ -module(emqx_sm_SUITE). +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -include("emqx.hrl"). -compile(export_all). -compile(nowarn_export_all). -all() -> [t_open_close_session]. +-define(ATTRS, #{clean_start => true, + client_id => <<"client">>, + zone => internal, + username => <<"emqx">>, + expiry_interval => 0, + max_inflight => 0, + topic_alias_maximum => 0, + will_msg => undefined}). + +all() -> [{group, sm}]. + +groups() -> + [{sm, [non_parallel_tests], + [t_open_close_session, + t_resume_session, + t_discard_session, + t_register_unregister_session, + t_get_set_session_attrs, + t_get_set_session_stats, + t_lookup_session_pids]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). t_open_close_session(_) -> - emqx_ct_broker_helpers:run_setup_steps(), {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), - Attrs = #{clean_start => true, - client_id => <<"client">>, - conn_pid => ClientPid, - zone => internal, - username => <<"emqx">>, - expiry_interval => 0, - max_inflight => 0, - topic_alias_maximum => 0, - will_msg => undefined}, - {ok, SPid} = emqx_sm:open_session(Attrs), - ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)), - {ok, NewConnPid} = emqx_mock_client:start_link(<<"client">>), - {ok, SPid, true} = emqx_sm:open_session(Attrs#{clean_start => false, conn_pid => NewConnPid}), - ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)), - SAttrs = emqx_sm:get_session_attrs(<<"client">>, SPid), - ?assertEqual(<<"client">>, proplists:get_value(client_id, SAttrs)), - emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}]), - ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)), - ok = emqx_sm:close_session(SPid), - ?assertEqual([], emqx_sm:lookup_session_pids(<<"client">>)), - emqx_ct_broker_helpers:run_teardown_steps(). + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:close_session(SPid)). + +t_resume_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual({ok, SPid}, emqx_sm:resume_session(<<"client">>, ?ATTRS#{conn_pid => ClientPid})). + +t_discard_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client1">>), + {ok, _SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:discard_session(<<"client1">>)). + +t_register_unregister_session(_) -> + Pid = self(), + {ok, _ClientPid} = emqx_mock_client:start_link(<<"client">>), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>, Pid)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>), Pid). + +t_get_set_session_attrs(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, [?ATTRS#{conn_pid => ClientPid}])), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, SPid, [?ATTRS#{conn_pid => ClientPid}])), + [SAttr] = emqx_sm:get_session_attrs(<<"client">>, SPid), + ?assertEqual(<<"client">>, maps:get(client_id, SAttr)). + +t_get_set_session_stats(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, [{inflight, 10}])), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}])), + ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)). + +t_lookup_session_pids(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)). From 1007105b5710dda931225461f9a52976ad880b3f Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Thu, 20 Dec 2018 22:29:31 +0800 Subject: [PATCH 47/54] Delete metrics test in session test suite --- test/emqx_session_SUITE.erl | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index 2e597340d..b8e0aedd3 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -57,17 +57,6 @@ t_session_all(_) -> emqx_session:publish(SPid, 1, Message1), timer:sleep(200), {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid), - emqx_session:puback(SPid, 2), - emqx_session:puback(SPid, 3, reasoncode), - emqx_session:pubrec(SPid, 4), - emqx_session:pubrec(SPid, 5, reasoncode), - emqx_session:pubrel(SPid, 6, reasoncode), - emqx_session:pubcomp(SPid, 7, reasoncode), - timer:sleep(200), - 2 = emqx_metrics:val('packets/puback/missed'), - 2 = emqx_metrics:val('packets/pubrec/missed'), - 1 = emqx_metrics:val('packets/pubrel/missed'), - 1 = emqx_metrics:val('packets/pubcomp/missed'), Attrs = emqx_session:attrs(SPid), Info = emqx_session:info(SPid), Stats = emqx_session:stats(SPid), From 367b717c4013ea22b2e8401fd43b95ea69bee842 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Thu, 20 Dec 2018 22:42:18 +0800 Subject: [PATCH 48/54] Implement a new 'emqx_gc' module (#2090) Update connection/session module to using the new emqx_gc API --- src/emqx_connection.erl | 42 +++++++++--------- src/emqx_gc.erl | 97 ++++++++++++++++++++++------------------- src/emqx_session.erl | 27 ++++++++---- 3 files changed, 92 insertions(+), 74 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 2cf647beb..ddb7e6a85 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -40,10 +40,10 @@ active_n, proto_state, parser_state, + gc_state, keepalive, enable_stats, stats_timer, - incoming, rate_limit, pub_limit, limit_timer, @@ -138,6 +138,8 @@ init([Transport, RawSocket, Options]) -> peercert => Peercert, sendfun => SendFun}, Options), ParserState = emqx_protocol:parser(ProtoState), + GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), + GcState = emqx_gc:init(GcPolicy), State = run_socket(#state{transport = Transport, socket = Socket, peername = Peername, @@ -147,11 +149,10 @@ init([Transport, RawSocket, Options]) -> pub_limit = PubLimit, proto_state = ProtoState, parser_state = ParserState, + gc_state = GcState, enable_stats = EnableStats, idle_timeout = IdleTimout }), - GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), - ok = emqx_gc:init(GcPolicy), ok = emqx_misc:init_proc_mng_policy(Zone), emqx_logger:set_metadata_peername(esockd_net:format(Peername)), gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], @@ -205,9 +206,8 @@ handle_cast(Msg, State) -> handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> case emqx_protocol:deliver(PubOrAck, ProtoState) of {ok, ProtoState1} -> - State1 = ensure_stats_timer(State#state{proto_state = ProtoState1}), - ok = maybe_gc(State1, PubOrAck), - {noreply, State1}; + State1 = State#state{proto_state = ProtoState1}, + {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))}; {error, Reason} -> shutdown(Reason, State) end; @@ -247,11 +247,10 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> handle_info({tcp, _Sock, Data}, State) -> ?LOG(debug, "RECV ~p", [Data]), - Size = iolist_size(Data), - emqx_metrics:trans(inc, 'bytes/received', Size), - emqx_pd:update_counter(incoming_bytes, Size), - Incoming = #{bytes => Size, packets => 0}, - handle_packet(Data, State#state{incoming = Incoming}); + Oct = iolist_size(Data), + emqx_pd:update_counter(incoming_bytes, Oct), + emqx_metrics:trans(inc, 'bytes/received', Oct), + handle_packet(Data, maybe_gc({1, Oct}, State)); %% Rate limit here, cool:) handle_info({tcp_passive, _Sock}, State) -> @@ -325,9 +324,7 @@ code_change(_OldVsn, State, _Extra) -> %% Receive and parse data handle_packet(<<>>, State) -> - NState = ensure_stats_timer(State), - ok = maybe_gc(NState, incoming), - {noreply, NState}; + {noreply, ensure_stats_timer(State)}; handle_packet(Data, State = #state{proto_state = ProtoState, parser_state = ParserState, @@ -407,14 +404,15 @@ shutdown(Reason, State) -> stop(Reason, State) -> {stop, Reason, State}. -%% For incoming messages, bump gc-stats with packet count and totoal volume -%% For outgoing messages, only 'publish' type is taken into account. -maybe_gc(#state{incoming = #{bytes := Oct, packets := Cnt}}, incoming) -> - ok = emqx_gc:inc(Cnt, Oct); -maybe_gc(#state{}, {publish, _PacketId, #message{payload = Payload}}) -> +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) -> Oct = iolist_size(Payload), - ok = emqx_gc:inc(1, Oct); -maybe_gc(_, _) -> - ok. + maybe_gc({1, Oct}, State); +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}; +maybe_gc(_, State) -> + State. diff --git a/src/emqx_gc.erl b/src/emqx_gc.erl index 7e98eb37a..d608954a0 100644 --- a/src/emqx_gc.erl +++ b/src/emqx_gc.erl @@ -21,74 +21,83 @@ -module(emqx_gc). --export([init/1, inc/2, reset/0]). +-export([init/1, run/3, info/1, reset/1]). --type st() :: #{ cnt => {integer(), integer()} - , oct => {integer(), integer()} - }. +-type(opts() :: #{count => integer(), + bytes => integer()}). + +-type(st() :: #{cnt => {integer(), integer()}, + oct => {integer(), integer()}}). + +-type(gc_state() :: {?MODULE, st()}). -define(disabled, disabled). -define(ENABLED(X), (is_integer(X) andalso X > 0)). -%% @doc Initialize force GC parameters. --spec init(false | map()) -> ok. +%% @doc Initialize force GC state. +-spec(init(opts() | false) -> gc_state() | undefined). init(#{count := Count, bytes := Bytes}) -> Cnt = [{cnt, {Count, Count}} || ?ENABLED(Count)], Oct = [{oct, {Bytes, Bytes}} || ?ENABLED(Bytes)], - erlang:put(?MODULE, maps:from_list(Cnt ++ Oct)), - ok; -init(_) -> erlang:put(?MODULE, #{}), ok. + {?MODULE, maps:from_list(Cnt ++ Oct)}; +init(false) -> undefined. -%% @doc Increase count and bytes stats in one call, -%% ensure gc is triggered at most once, even if both thresholds are hit. --spec inc(pos_integer(), pos_integer()) -> ok. -inc(Cnt, Oct) -> - mutate_pd_with(fun(St) -> inc(St, Cnt, Oct) end). +%% @doc Try to run GC based on reduntions of count or bytes. +-spec(run(pos_integer(), pos_integer(), gc_state()) -> {boolean(), gc_state()}). +run(Cnt, Oct, {?MODULE, St}) -> + {Res, St1} = run([{cnt, Cnt}, {oct, Oct}], St), + {Res, {?MODULE, St1}}; +run(_Cnt, _Oct, undefined) -> + {false, undefined}. -%% @doc Reset counters to zero. --spec reset() -> ok. -reset() -> - mutate_pd_with(fun(St) -> reset(St) end). - -%% ======== Internals ======== - -%% mutate gc stats numbers in process dict with the given function -mutate_pd_with(F) -> - St = F(erlang:get(?MODULE)), - erlang:put(?MODULE, St), - ok. - -%% Increase count and bytes stats in one call, -%% ensure gc is triggered at most once, even if both thresholds are hit. --spec inc(st(), pos_integer(), pos_integer()) -> st(). -inc(St0, Cnt, Oct) -> - case do_inc(St0, cnt, Cnt) of - {true, St} -> - St; +run([], St) -> + {false, St}; +run([{K, N}|T], St) -> + case dec(K, N, St) of + {true, St1} -> + {true, do_gc(St1)}; {false, St1} -> - {_, St} = do_inc(St1, oct, Oct), - St + run(T, St1) end. -%% Reset counters to zero. -reset(St) -> reset(cnt, reset(oct, St)). +%% @doc Info of GC state. +-spec(info(gc_state()) -> map() | undefined). +info({?MODULE, St}) -> + St; +info(undefined) -> + undefined. --spec do_inc(st(), cnt | oct, pos_integer()) -> {boolean(), st()}. -do_inc(St, Key, Num) -> +%% @doc Reset counters to zero. +-spec(reset(gc_state()) -> gc_state()). +reset({?MODULE, St}) -> + {?MODULE, do_reset(St)}; +reset(undefined) -> + undefined. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +-spec(dec(cnt | oct, pos_integer(), st()) -> {boolean(), st()}). +dec(Key, Num, St) -> case maps:get(Key, St, ?disabled) of ?disabled -> {false, St}; {Init, Remain} when Remain > Num -> {false, maps:put(Key, {Init, Remain - Num}, St)}; _ -> - {true, do_gc(St)} + {true, St} end. do_gc(St) -> - erlang:garbage_collect(), - reset(St). + true = erlang:garbage_collect(), + do_reset(St). -reset(Key, St) -> +do_reset(St) -> + do_reset(cnt, do_reset(oct, St)). + +%% Reset counters to zero. +do_reset(Key, St) -> case maps:get(Key, St, ?disabled) of ?disabled -> St; {Init, _} -> maps:put(Key, {Init, Init}, St) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index e9bc0ff98..f4dbc2b23 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -144,6 +144,9 @@ %% Enqueue stats enqueue_stats = 0, + %% GC State + gc_state, + %% Created at created_at :: erlang:timestamp(), @@ -344,6 +347,7 @@ init([Parent, #{zone := Zone, process_flag(trap_exit, true), true = link(ConnPid), emqx_logger:set_metadata_client_id(ClientId), + GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), IdleTimout = get_env(Zone, idle_timeout, 30000), State = #state{idle_timeout = IdleTimout, clean_start = CleanStart, @@ -364,6 +368,7 @@ init([Parent, #{zone := Zone, enable_stats = get_env(Zone, enable_stats, true), deliver_stats = 0, enqueue_stats = 0, + gc_state = emqx_gc:init(GcPolicy), created_at = os:timestamp(), will_msg = WillMsg }, @@ -371,8 +376,6 @@ init([Parent, #{zone := Zone, true = emqx_sm:set_session_attrs(ClientId, attrs(State)), true = emqx_sm:set_session_stats(ClientId, stats(State)), emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), - GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), - ok = emqx_gc:init(GcPolicy), ok = emqx_misc:init_proc_mng_policy(Zone), ok = proc_lib:init_ack(Parent, {ok, self()}), gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State). @@ -605,7 +608,9 @@ handle_info({timeout, Timer, check_awaiting_rel}, State = #state{await_rel_timer noreply(ensure_stats_timer(expire_awaiting_rel(State1))); handle_info({timeout, Timer, emit_stats}, - State = #state{client_id = ClientId, stats_timer = Timer}) -> + State = #state{client_id = ClientId, + stats_timer = Timer, + gc_state = GcState}) -> emqx_metrics:commit(), _ = emqx_sm:set_session_stats(ClientId, stats(State)), NewState = State#state{stats_timer = undefined}, @@ -614,8 +619,9 @@ handle_info({timeout, Timer, emit_stats}, continue -> {noreply, NewState}; hibernate -> - ok = emqx_gc:reset(), %% going to hibernate, reset gc stats - {noreply, NewState, hibernate}; + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; {shutdown, Reason} -> ?LOG(warning, "shutdown due to ~p", [Reason], NewState), shutdown(Reason, NewState) @@ -991,9 +997,8 @@ next_pkt_id(State = #state{next_pkt_id = Id}) -> %% Inc stats inc_stats(deliver, Msg, State = #state{deliver_stats = I}) -> - MsgSize = msg_size(Msg), - ok = emqx_gc:inc(1, MsgSize), - State#state{deliver_stats = I + 1}; + State1 = maybe_gc({1, msg_size(Msg)}, State), + State1#state{deliver_stats = I + 1}; inc_stats(enqueue, _Msg, State = #state{enqueue_stats = I}) -> State#state{enqueue_stats = I + 1}. @@ -1018,3 +1023,9 @@ noreply(State) -> shutdown(Reason, State) -> {stop, Reason, State}. +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}. + From c93d0fb17426fb39502d071bc98031d53ef2b97e Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 21 Dec 2018 10:16:45 +0800 Subject: [PATCH 49/54] Add test cases for emqx_gc module --- Makefile | 2 +- test/emqx_gc_SUITE.erl | 57 ++++++++++++++++++++++++++++++++++++++++++ test/emqx_gc_tests.erl | 53 --------------------------------------- 3 files changed, 58 insertions(+), 54 deletions(-) create mode 100644 test/emqx_gc_SUITE.erl delete mode 100644 test/emqx_gc_tests.erl diff --git a/Makefile b/Makefile index 374b74fb7..758e93a05 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ - emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd + emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/test/emqx_gc_SUITE.erl b/test/emqx_gc_SUITE.erl new file mode 100644 index 000000000..22d7cd584 --- /dev/null +++ b/test/emqx_gc_SUITE.erl @@ -0,0 +1,57 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_init, t_run, t_info, t_reset]. + +t_init(_) -> + ?assertEqual(undefined, emqx_gc:init(false)), + GC1 = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC1)), + GC2 = emqx_gc:init(#{count => 0, bytes => 10}), + ?assertEqual(#{oct => {10, 10}}, emqx_gc:info(GC2)), + GC3 = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual(#{cnt => {10, 10}, oct => {10, 10}}, emqx_gc:info(GC3)). + +t_run(_) -> + GC = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual({true, GC}, emqx_gc:run(1, 1000, GC)), + ?assertEqual({true, GC}, emqx_gc:run(1000, 1, GC)), + {false, GC1} = emqx_gc:run(1, 1, GC), + ?assertEqual(#{cnt => {10, 9}, oct => {10, 9}}, emqx_gc:info(GC1)), + {false, GC2} = emqx_gc:run(2, 2, GC1), + ?assertEqual(#{cnt => {10, 7}, oct => {10, 7}}, emqx_gc:info(GC2)), + {false, GC3} = emqx_gc:run(3, 3, GC2), + ?assertEqual(#{cnt => {10, 4}, oct => {10, 4}}, emqx_gc:info(GC3)), + ?assertEqual({true, GC}, emqx_gc:run(4, 4, GC3)). + +t_info(_) -> + ?assertEqual(undefined, emqx_gc:info(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC)). + +t_reset(_) -> + ?assertEqual(undefined, emqx_gc:reset(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 10}), + {false, GC1} = emqx_gc:run(5, 5, GC), + ?assertEqual(#{cnt => {10, 5}, oct => {10, 5}}, emqx_gc:info(GC1)), + ?assertEqual(GC, emqx_gc:reset(GC1)). + diff --git a/test/emqx_gc_tests.erl b/test/emqx_gc_tests.erl deleted file mode 100644 index ffcac91d1..000000000 --- a/test/emqx_gc_tests.erl +++ /dev/null @@ -1,53 +0,0 @@ -%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_gc_tests). - --include_lib("eunit/include/eunit.hrl"). - -trigger_by_cnt_test() -> - Args = #{count => 2, bytes => 0}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1000), - St1 = inspect(), - ?assertMatch({_, Remain} when Remain > 0, maps:get(cnt, St1)), - ok = emqx_gc:inc(2, 2), - St2 = inspect(), - ok = emqx_gc:inc(0, 2000), - St3 = inspect(), - ?assertEqual(St2, St3), - ?assertMatch({N, N}, maps:get(cnt, St2)), - ?assertNot(maps:is_key(oct, St2)), - ok. - -trigger_by_oct_test() -> - Args = #{count => 2, bytes => 2}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1), - St1 = inspect(), - ?assertMatch({_, Remain} when Remain > 0, maps:get(oct, St1)), - ok = emqx_gc:inc(2, 2), - St2 = inspect(), - ?assertMatch({N, N}, maps:get(oct, St2)), - ?assertMatch({M, M}, maps:get(cnt, St2)), - ok. - -disabled_test() -> - Args = #{count => -1, bytes => false}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1), - ?assertEqual(#{}, inspect()), - ok. - -inspect() -> erlang:get(emqx_gc). From 4f84a31d0283482af54d4b13aa27557b9e006541 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Fri, 21 Dec 2018 11:06:50 +0800 Subject: [PATCH 50/54] Update copyright (#2093) --- src/emqx_cm_sup.erl | 1 - src/emqx_mqueue.erl | 3 ++- src/emqx_plugins.erl | 28 +++++++++++++--------------- src/emqx_rate_limiter.erl | 28 +++++++++++++--------------- src/emqx_sm_sup.erl | 1 - src/emqx_vm.erl | 4 +--- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/emqx_cm_sup.erl b/src/emqx_cm_sup.erl index 19dd9cb50..e8c99c16b 100644 --- a/src/emqx_cm_sup.erl +++ b/src/emqx_cm_sup.erl @@ -1,6 +1,5 @@ %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% -%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl index 56390d412..48b0fa439 100644 --- a/src/emqx_mqueue.erl +++ b/src/emqx_mqueue.erl @@ -5,7 +5,8 @@ %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 -%%%% Unless required by applicable law or agreed to in writing, software +%% +%% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 44585a69e..b0f6456dc 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -1,18 +1,16 @@ -%%%=================================================================== -%%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved. -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%%=================================================================== +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. -module(emqx_plugins). diff --git a/src/emqx_rate_limiter.erl b/src/emqx_rate_limiter.erl index 76eaf0c61..c8145dbbc 100644 --- a/src/emqx_rate_limiter.erl +++ b/src/emqx_rate_limiter.erl @@ -1,18 +1,16 @@ -%%%------------------------------------------------------------------- -%%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%%------------------------------------------------------------------- +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. -module(emqx_rate_limiter). diff --git a/src/emqx_sm_sup.erl b/src/emqx_sm_sup.erl index 317cd11db..ed491c67f 100644 --- a/src/emqx_sm_sup.erl +++ b/src/emqx_sm_sup.erl @@ -1,6 +1,5 @@ %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% -%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at diff --git a/src/emqx_vm.erl b/src/emqx_vm.erl index bf6388232..74b815795 100644 --- a/src/emqx_vm.erl +++ b/src/emqx_vm.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved. +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,7 +11,6 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -module(emqx_vm). From 3fec9cdf0a792ad113cac237d80313b3bf8362c6 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 21 Dec 2018 11:56:22 +0800 Subject: [PATCH 51/54] Try to simulate a '{ssl_passive, Sock}' message:( --- src/emqx_connection.erl | 54 +++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index ddb7e6a85..1714632a8 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -214,8 +214,8 @@ handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> handle_info({timeout, Timer, emit_stats}, State = #state{stats_timer = Timer, - proto_state = ProtoState - }) -> + proto_state = ProtoState, + gc_state = GcState}) -> emqx_metrics:commit(), emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), NewState = State#state{stats_timer = undefined}, @@ -224,8 +224,9 @@ handle_info({timeout, Timer, emit_stats}, continue -> {noreply, NewState}; hibernate -> - ok = emqx_gc:reset(), - {noreply, NewState, hibernate}; + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; {shutdown, Reason} -> ?LOG(warning, "shutdown due to ~p", [Reason]), shutdown(Reason, NewState) @@ -246,22 +247,29 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> shutdown(conflict, State); handle_info({tcp, _Sock, Data}, State) -> - ?LOG(debug, "RECV ~p", [Data]), - Oct = iolist_size(Data), - emqx_pd:update_counter(incoming_bytes, Oct), - emqx_metrics:trans(inc, 'bytes/received', Oct), - handle_packet(Data, maybe_gc({1, Oct}, State)); + process_incoming(Data, State); +%% FIXME Later +handle_info({ssl, _Sock, Data}, State) -> + process_incoming(Data, run_socket(State)); %% Rate limit here, cool:) handle_info({tcp_passive, _Sock}, State) -> {noreply, run_socket(ensure_rate_limit(State))}; +%% FIXME Later +handle_info({ssl_passive, _Sock}, State) -> + {noreply, run_socket(ensure_rate_limit(State))}; handle_info({tcp_error, _Sock, Reason}, State) -> shutdown(Reason, State); +handle_info({ssl_error, _Sock, Reason}, State) -> + shutdown(Reason, State); handle_info({tcp_closed, _Sock}, State) -> shutdown(closed, State); +handle_info({ssl_closed, _Sock}, State) -> + shutdown(closed, State); +%% Rate limit timer handle_info(activate_sock, State) -> {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; @@ -319,12 +327,24 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. %%------------------------------------------------------------------------------ -%% Parse and handle packets +%% Internals: process incoming, parse and handle packets %%------------------------------------------------------------------------------ -%% Receive and parse data +process_incoming(Data, State) -> + Oct = iolist_size(Data), + ?LOG(debug, "RECV ~p", [Data]), + emqx_pd:update_counter(incoming_bytes, Oct), + emqx_metrics:trans(inc, 'bytes/received', Oct), + case handle_packet(Data, State) of + {noreply, State1} -> + State2 = maybe_gc({1, Oct}, State1), + {noreply, ensure_stats_timer(State2)}; + Shutdown -> Shutdown + end. + +%% Parse and handle packets handle_packet(<<>>, State) -> - {noreply, ensure_stats_timer(State)}; + {noreply, State}; handle_packet(Data, State = #state{proto_state = ProtoState, parser_state = ParserState, @@ -384,8 +404,8 @@ run_socket(State = #state{conn_state = blocked}) -> State; run_socket(State = #state{transport = Transport, socket = Socket, - active_n = ActiveN}) -> - Transport:setopts(Socket, [{active, ActiveN}]), + active_n = N}) -> + ensure_ok_or_exit(Transport:setopts(Socket, [{active, N}])), State. %%------------------------------------------------------------------------------ @@ -393,7 +413,7 @@ run_socket(State = #state{transport = Transport, %%------------------------------------------------------------------------------ ensure_stats_timer(State = #state{enable_stats = true, - stats_timer = undefined, + stats_timer = undefined, idle_timeout = IdleTimeout}) -> State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; ensure_stats_timer(State) -> State. @@ -415,4 +435,8 @@ maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> maybe_gc(_, State) -> State. +ensure_ok_or_exit(ok) -> + ok; +ensure_ok_or_exit({error, Reason}) -> + self() ! {shutdown, Reason}. From 31bf01fd7ab11240dd74c7cd85b6470add3c16d3 Mon Sep 17 00:00:00 2001 From: tigercl Date: Fri, 21 Dec 2018 14:01:21 +0800 Subject: [PATCH 52/54] Fix bug in topic alias maximum (#2074) * Fix bug in topic alias maximum --- src/emqx_protocol.erl | 133 +++++++++++++++++++++-------------- test/emqx_protocol_SUITE.erl | 77 ++++++++++++++++++++ 2 files changed, 159 insertions(+), 51 deletions(-) diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 8f845a94e..d3e0e9f97 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -64,7 +64,8 @@ send_stats, connected, connected_at, - ignore_loop + ignore_loop, + topic_alias_maximum }). -type(state() :: #pstate{}). @@ -84,28 +85,29 @@ -spec(init(map(), list()) -> state()). init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) -> Zone = proplists:get_value(zone, Options), - #pstate{zone = Zone, - sendfun = SendFun, - peername = Peername, - peercert = Peercert, - proto_ver = ?MQTT_PROTO_V4, - proto_name = <<"MQTT">>, - client_id = <<>>, - is_assigned = false, - conn_pid = self(), - username = init_username(Peercert, Options), - is_super = false, - clean_start = false, - topic_aliases = #{}, - packet_size = emqx_zone:get_env(Zone, max_packet_size), - mountpoint = emqx_zone:get_env(Zone, mountpoint), - is_bridge = false, - enable_ban = emqx_zone:get_env(Zone, enable_ban, false), - enable_acl = emqx_zone:get_env(Zone, enable_acl), - recv_stats = #{msg => 0, pkt => 0}, - send_stats = #{msg => 0, pkt => 0}, - connected = false, - ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false)}. + #pstate{zone = Zone, + sendfun = SendFun, + peername = Peername, + peercert = Peercert, + proto_ver = ?MQTT_PROTO_V4, + proto_name = <<"MQTT">>, + client_id = <<>>, + is_assigned = false, + conn_pid = self(), + username = init_username(Peercert, Options), + is_super = false, + clean_start = false, + topic_aliases = #{}, + packet_size = emqx_zone:get_env(Zone, max_packet_size), + mountpoint = emqx_zone:get_env(Zone, mountpoint), + is_bridge = false, + enable_ban = emqx_zone:get_env(Zone, enable_ban, false), + enable_acl = emqx_zone:get_env(Zone, enable_acl), + recv_stats = #{msg => 0, pkt => 0}, + send_stats = #{msg => 0, pkt => 0}, + connected = false, + ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false), + topic_alias_maximum = #{to_client => 0, from_client => 0}}. init_username(Peercert, Options) -> case proplists:get_value(peer_cert_as_username, Options) of @@ -212,12 +214,16 @@ received(?PACKET(?CONNECT), PState = #pstate{connected = true}) -> {error, proto_unexpected_connect, PState}; received(Packet = ?PACKET(Type), PState) -> - PState1 = set_protover(Packet, PState), + PState1 = set_protover(Packet, PState), trace(recv, Packet), try emqx_packet:validate(Packet) of true -> - {Packet1, PState2} = preprocess_properties(Packet, PState1), - process_packet(Packet1, inc_stats(recv, Type, PState2)) + case preprocess_properties(Packet, PState1) of + {error, ReasonCode} -> + {error, ReasonCode, PState1}; + {Packet1, PState2} -> + process_packet(Packet1, inc_stats(recv, Type, PState2)) + end catch error : protocol_error -> deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState1), @@ -242,6 +248,13 @@ received(Packet = ?PACKET(Type), PState) -> %%------------------------------------------------------------------------------ %% Preprocess MQTT Properties %%------------------------------------------------------------------------------ +preprocess_properties(Packet = #mqtt_packet{ + variable = #mqtt_packet_connect{ + properties = #{'Topic-Alias-Maximum' := ToClient} + } + }, + PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) -> + {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}}; %% Subscription Identifier preprocess_properties(Packet = #mqtt_packet{ @@ -255,22 +268,46 @@ preprocess_properties(Packet = #mqtt_packet{ {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState}; %% Topic Alias Mapping +preprocess_properties(#mqtt_packet{ + variable = #mqtt_packet_publish{ + properties = #{'Topic-Alias' := 0}} + }, + PState) -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID}; + preprocess_properties(Packet = #mqtt_packet{ variable = Publish = #mqtt_packet_publish{ topic_name = <<>>, properties = #{'Topic-Alias' := AliasId}} }, - PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) -> - {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ - topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ + topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; preprocess_properties(Packet = #mqtt_packet{ - variable = #mqtt_packet_publish{ - topic_name = Topic, - properties = #{'Topic-Alias' := AliasId}} - }, - PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) -> - {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + variable = #mqtt_packet_publish{ + topic_name = Topic, + properties = #{'Topic-Alias' := AliasId}} + }, + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; preprocess_properties(Packet, PState) -> {Packet, PState}. @@ -278,7 +315,6 @@ preprocess_properties(Packet, PState) -> %%------------------------------------------------------------------------------ %% Process MQTT Packet %%------------------------------------------------------------------------------ - process_packet(?CONNECT_PACKET( #mqtt_packet_connect{proto_name = ProtoName, proto_ver = ProtoVer, @@ -308,6 +344,7 @@ process_packet(?CONNECT_PACKET( will_msg = WillMsg, is_bridge = IsBridge, connected_at = os:timestamp()}), + connack( case check_connect(Connect, PState1) of {ok, PState2} -> @@ -342,9 +379,6 @@ process_packet(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PSt case check_publish(Packet, PState) of {ok, PState1} -> do_publish(Packet, PState1); - {error, ?RC_TOPIC_ALIAS_INVALID} -> - ?LOG(error, "Protocol error - ~p", [?RC_TOPIC_ALIAS_INVALID]), - {error, ?RC_TOPIC_ALIAS_INVALID, PState}; {error, ReasonCode} -> ?LOG(warning, "Cannot publish qos0 message to ~s for ~s", [Topic, emqx_reason_codes:text(ReasonCode)]), @@ -523,7 +557,8 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone, proto_ver = ?MQTT_PROTO_V5, client_id = ClientId, conn_props = ConnProps, - is_assigned = IsAssigned}) -> + is_assigned = IsAssigned, + topic_alias_maximum = TopicAliasMaximum}) -> ResponseInformation = case maps:find('Request-Response-Information', ConnProps) of {ok, 1} -> iolist_to_binary(emqx_config:get_env(response_topic_prefix)); @@ -561,17 +596,20 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone, undefined -> Props2; Keepalive -> Props2#{'Server-Keep-Alive' => Keepalive} end, - send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState); + + PState1 = PState#pstate{topic_alias_maximum = TopicAliasMaximum#{from_client => MaxAlias}}, + + send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState1); deliver({connack, ReasonCode, SP}, PState) -> send(?CONNACK_PACKET(ReasonCode, SP), PState); -deliver({publish, PacketId, Msg}, PState = #pstate{mountpoint = MountPoint}) -> +deliver({publish, PacketId, Msg = #message{headers = Headers}}, PState = #pstate{mountpoint = MountPoint}) -> _ = emqx_hooks:run('message.delivered', [credentials(PState)], Msg), Msg1 = emqx_message:update_expiry(Msg), Msg2 = emqx_mountpoint:unmount(MountPoint, Msg1), - send(emqx_packet:from_message(PacketId, Msg2), PState); - + send(emqx_packet:from_message(PacketId, Msg2#message{headers = maps:remove('Topic-Alias', Headers)}), PState); + deliver({puback, PacketId, ReasonCode}, PState) -> send(?PUBACK_PACKET(PacketId, ReasonCode), PState); @@ -758,18 +796,11 @@ check_publish(Packet, PState) -> run_check_steps([fun check_pub_caps/2, fun check_pub_acl/2], Packet, PState). -check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ - properties = #{'Topic-Alias' := TopicAlias} - }}, - #pstate{zone = Zone}) -> - emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}); check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{ properties = _Properties}}, #pstate{zone = Zone}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}). - check_pub_acl(_Packet, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) when IsSuper orelse (not EnableAcl) -> ok; diff --git a/test/emqx_protocol_SUITE.erl b/test/emqx_protocol_SUITE.erl index ae308ea42..27137d956 100644 --- a/test/emqx_protocol_SUITE.erl +++ b/test/emqx_protocol_SUITE.erl @@ -55,6 +55,7 @@ groups() -> init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), + emqx_zone:set_env(external, max_topic_alias, 20), Config. end_per_suite(_Config) -> @@ -154,6 +155,82 @@ connect_v5(_) -> #{'Response-Information' := _RespInfo}), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5) end), + + % topic alias = 0 + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 0}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5) + end), + + % topic alias maximum + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 15}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data3} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBACK_PACKET(1, 0), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5), + + {ok, Data4} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, _, <<"hello">>), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 2, #{'Topic-Alias' => 21}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data5} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data5, ?MQTT_PROTO_V5) + end), % test clean start with_connection(fun([Sock]) -> From 10e5210581ea0097ebc0c28e811135c3a449d63c Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Fri, 21 Dec 2018 15:39:24 +0800 Subject: [PATCH 53/54] Workaround ssl:setopts(SslSock, [{active, N}]) (#2095) * Set '{active, true}' for SSL socket --- src/emqx_connection.erl | 51 ++++++++++++++++++++------------------- test/emqx_stats_tests.erl | 4 +-- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 1714632a8..9f2572b32 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -246,11 +246,8 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), shutdown(conflict, State); -handle_info({tcp, _Sock, Data}, State) -> +handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> process_incoming(Data, State); -%% FIXME Later -handle_info({ssl, _Sock, Data}, State) -> - process_incoming(Data, run_socket(State)); %% Rate limit here, cool:) handle_info({tcp_passive, _Sock}, State) -> @@ -259,14 +256,10 @@ handle_info({tcp_passive, _Sock}, State) -> handle_info({ssl_passive, _Sock}, State) -> {noreply, run_socket(ensure_rate_limit(State))}; -handle_info({tcp_error, _Sock, Reason}, State) -> - shutdown(Reason, State); -handle_info({ssl_error, _Sock, Reason}, State) -> +handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error -> shutdown(Reason, State); -handle_info({tcp_closed, _Sock}, State) -> - shutdown(closed, State); -handle_info({ssl_closed, _Sock}, State) -> +handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed -> shutdown(closed, State); %% Rate limit timer @@ -380,7 +373,6 @@ reset_parser(State = #state{proto_state = ProtoState}) -> %%------------------------------------------------------------------------------ %% Ensure rate limit -%%------------------------------------------------------------------------------ ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl}) -> Limiters = [{Pl, #state.pub_limit, emqx_pd:reset_counter(incoming_pubs)}, @@ -400,17 +392,26 @@ ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) -> setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1) end. +%%------------------------------------------------------------------------------ +%% Activate socket + run_socket(State = #state{conn_state = blocked}) -> State; -run_socket(State = #state{transport = Transport, - socket = Socket, - active_n = N}) -> - ensure_ok_or_exit(Transport:setopts(Socket, [{active, N}])), + +run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) -> + TrueOrN = case Transport:is_ssl(Socket) of + true -> true; %% Cannot set '{active, N}' for SSL:( + false -> N + end, + ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])), State. +ensure_ok_or_exit(ok) -> ok; +ensure_ok_or_exit({error, Reason}) -> + self() ! {shutdown, Reason}. + %%------------------------------------------------------------------------------ %% Ensure stats timer -%%------------------------------------------------------------------------------ ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined, @@ -418,11 +419,8 @@ ensure_stats_timer(State = #state{enable_stats = true, State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; ensure_stats_timer(State) -> State. -shutdown(Reason, State) -> - stop({shutdown, Reason}, State). - -stop(Reason, State) -> - {stop, Reason, State}. +%%------------------------------------------------------------------------------ +%% Maybe GC maybe_gc(_, State = #state{gc_state = undefined}) -> State; @@ -435,8 +433,11 @@ maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> maybe_gc(_, State) -> State. -ensure_ok_or_exit(ok) -> - ok; -ensure_ok_or_exit({error, Reason}) -> - self() ! {shutdown, Reason}. +%%------------------------------------------------------------------------------ +%% Shutdown or stop +shutdown(Reason, State) -> + stop({shutdown, Reason}, State). + +stop(Reason, State) -> + {stop, Reason, State}. diff --git a/test/emqx_stats_tests.erl b/test/emqx_stats_tests.erl index dd9733a88..e8b5e82af 100644 --- a/test/emqx_stats_tests.erl +++ b/test/emqx_stats_tests.erl @@ -75,10 +75,10 @@ helper_test_() -> with_proc(fun() -> TestF(CbModule, CbFun) end, TickMs) end end, - [{"emqx_broker_helper", MkTestFun(emqx_broker_helper, stats_fun)}, + [{"emqx_broker", MkTestFun(emqx_broker, stats_fun)}, {"emqx_sm", MkTestFun(emqx_sm, stats_fun)}, {"emqx_router_helper", MkTestFun(emqx_router_helper, stats_fun)}, - {"emqx_cm", MkTestFun(emqx_cm, update_conn_stats)} + {"emqx_cm", MkTestFun(emqx_cm, stats_fun)} ]. with_proc(F) -> From e949e8cbd80ad41f305ea857b816c3feb95885e2 Mon Sep 17 00:00:00 2001 From: YoukiLin <1045735402@qq.com> Date: Fri, 21 Dec 2018 15:42:42 +0800 Subject: [PATCH 54/54] Add format output for test print (#2076) * Add formatted output for the test print --- src/emqx_cli.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/emqx_cli.erl b/src/emqx_cli.erl index 6be9093b5..f7d513e9d 100644 --- a/src/emqx_cli.erl +++ b/src/emqx_cli.erl @@ -17,16 +17,17 @@ -export([print/1, print/2, usage/1, usage/2]). print(Msg) -> - io:format(Msg). + io:format(Msg), lists:flatten(io_lib:format("~p", [Msg])). print(Format, Args) -> - io:format(Format, Args). + io:format(Format, Args), lists:flatten(io_lib:format(Format, Args)). usage(CmdList) -> - lists:foreach( + lists:map( fun({Cmd, Descr}) -> - io:format("~-48s# ~s~n", [Cmd, Descr]) + io:format("~-48s# ~s~n", [Cmd, Descr]), + lists:flatten(io_lib:format("~-48s# ~s~n", [Cmd, Descr])) end, CmdList). usage(Format, Args) -> - usage([{Format, Args}]). \ No newline at end of file + usage([{Format, Args}]).