diff --git a/Makefile b/Makefile index 26bcf22ce..758e93a05 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ NO_AUTOPATCH = cuttlefish ERLC_OPTS += +debug_info -DAPPLICATION=emqx BUILD_DEPS = cuttlefish -dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.1.1 +dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.0 #TEST_DEPS = emqx_ct_helplers #dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers @@ -36,7 +36,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ - emqx_hooks emqx_batch + emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/etc/emqx.conf b/etc/emqx.conf index 430c58ce4..cb713c4d0 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -160,11 +160,6 @@ node.name = emqx@127.0.0.1 ## Value: String node.cookie = emqxsecretcookie -## Enable SMP support of Erlang VM. -## -## Value: enable | auto | disable -node.smp = auto - ## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable ## heartbeat, or set the value as 'on' ## @@ -173,13 +168,6 @@ node.smp = auto ## vm.args: -heart ## node.heartbeat = on -## Enable kernel poll. -## -## Value: on | off -## -## Default: on -node.kernel_poll = on - ## Sets the number of threads in async thread pool. Valid range is 0-1024. ## ## See: http://erlang.org/doc/man/erl.html @@ -768,6 +756,11 @@ listener.tcp.external.max_connections = 1024000 ## Value: Number listener.tcp.external.max_conn_rate = 1000 +## Specify the {active, N} option for the external MQTT/TCP Socket. +## +## Value: Number +listener.tcp.external.active_n = 100 + ## Zone of the external MQTT/TCP listener belonged to. ## ## See: zone.$name.* @@ -904,6 +897,11 @@ listener.tcp.internal.max_connections = 1024000 ## Value: Number listener.tcp.internal.max_conn_rate = 1000 +## Specify the {active, N} option for the internal MQTT/TCP Socket. +## +## Value: Number +listener.tcp.internal.active_n = 1000 + ## Zone of the internal MQTT/TCP listener belonged to. ## ## Value: String @@ -1011,6 +1009,11 @@ listener.ssl.external.max_connections = 102400 ## Value: Number listener.ssl.external.max_conn_rate = 500 +## Specify the {active, N} option for the internal MQTT/SSL Socket. +## +## Value: Number +listener.ssl.external.active_n = 100 + ## Zone of the external MQTT/SSL listener belonged to. ## ## Value: String @@ -1916,6 +1919,11 @@ plugins.expand_plugins_dir = {{ platform_plugins_dir }}/ ## Default: 1m, 1 minute broker.sys_interval = 1m +## Enable global session registry. +## +## Value: on | off +broker.enable_session_registry = on + ## Session locking strategy in a cluster. ## ## Value: Enum diff --git a/etc/vm.args b/etc/vm.args new file mode 100644 index 000000000..43e6467d9 --- /dev/null +++ b/etc/vm.args @@ -0,0 +1,95 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. + +## Sets the maximum number of simultaneously existing processes for this system. +#+P 2048000 + +## Sets the maximum number of simultaneously existing ports for this system. +#+Q 1024000 + +## Sets the maximum number of ETS tables +#+e 256000 + +## Sets the maximum number of atoms the virtual machine can handle. +#+t 1048576 + +## Set the location of crash dumps +#-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +#-env ERL_FULLSWEEP_AFTER 1000 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). +#+zdbbl 8192 + +## Sets default scheduler hint for port parallelism. ++spp true + +## Sets the number of threads in async thread pool. Valid range is 0-1024. +#+A 8 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. +#+IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. +#+IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. +#+S 8:8 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. +#+SDcpu 8:8 + +## Sets the number of dirty I/O scheduler threads to create. +#+SDio 10 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/etc/vm.args.edge b/etc/vm.args.edge new file mode 100644 index 000000000..20adc41ab --- /dev/null +++ b/etc/vm.args.edge @@ -0,0 +1,95 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. + +## Sets the maximum number of simultaneously existing processes for this system. ++P 20480 + +## Sets the maximum number of simultaneously existing ports for this system. ++Q 4096 + +## Sets the maximum number of ETS tables ++e 512 + +## Sets the maximum number of atoms the virtual machine can handle. ++t 65536 + +## Set the location of crash dumps +-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +-env ERL_FULLSWEEP_AFTER 0 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). ++zdbbl 1024 + +## Sets default scheduler hint for port parallelism. ++spp false + +## Sets the number of threads in async thread pool. Valid range is 0-1024. ++A 1 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. ++IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. ++IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. ++S 1:1 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. ++SDcpu 1:1 + +## Sets the number of dirty I/O scheduler threads to create. ++SDio 1 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/priv/emqx.schema b/priv/emqx.schema index d59c8af24..f74b53e7f 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -191,13 +191,6 @@ end}. {default, "emqxsecretcookie"} ]}. -%% @doc SMP Support -{mapping, "node.smp", "vm_args.-smp", [ - {default, auto}, - {datatype, {enum, [enable, auto, disable]}}, - hidden -]}. - %% @doc http://erlang.org/doc/man/heart.html {mapping, "node.heartbeat", "vm_args.-heart", [ {datatype, flag}, @@ -211,13 +204,6 @@ end}. end end}. -%% @doc Enable Kernel Poll -{mapping, "node.kernel_poll", "vm_args.+K", [ - {default, on}, - {datatype, flag}, - hidden -]}. - %% @doc More information at: http://erlang.org/doc/man/erl.html {mapping, "node.async_threads", "vm_args.+A", [ {default, 64}, @@ -912,6 +898,11 @@ end}. {datatype, integer} ]}. +{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + {mapping, "listener.tcp.$name.zone", "emqx.listeners", [ {datatype, string} ]}. @@ -1007,6 +998,11 @@ end}. {datatype, integer} ]}. +{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + {mapping, "listener.ssl.$name.zone", "emqx.listeners", [ {datatype, string} ]}. @@ -1423,6 +1419,7 @@ end}. {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, + {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, {rate_limit, Ratelimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, @@ -1735,6 +1732,11 @@ end}. {default, "1m"} ]}. +{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ + {default, on}, + {datatype, flag} +]}. + {mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ {default, quorum}, {datatype, {enum, [local,one,quorum,all]}} diff --git a/rebar.config b/rebar.config index 424505d49..c3baebadb 100644 --- a/rebar.config +++ b/rebar.config @@ -9,7 +9,7 @@ {ekka, "v0.5.1"}, {clique, "develop"}, {esockd, "v5.4.2"}, - {cuttlefish, "v2.1.1"} + {cuttlefish, "v2.2.0"} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/src/emqx.erl b/src/emqx.erl index 72f1d6f81..76e966a59 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -22,11 +22,10 @@ %% PubSub API -export([subscribe/1, subscribe/2, subscribe/3]). -export([publish/1]). --export([unsubscribe/1, unsubscribe/2]). +-export([unsubscribe/1]). %% PubSub management API -export([topics/0, subscriptions/1, subscribers/1, subscribed/2]). --export([get_subopts/2, set_subopts/3]). %% Hooks API -export([hook/2, hook/3, hook/4, unhook/2, run_hooks/2, run_hooks/3]). @@ -70,20 +69,18 @@ is_running(Node) -> subscribe(Topic) -> emqx_broker:subscribe(iolist_to_binary(Topic)). --spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok). +-spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId)-> emqx_broker:subscribe(iolist_to_binary(Topic), SubId); -subscribe(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:subscribe(iolist_to_binary(Topic), SubPid). +subscribe(Topic, SubOpts) when is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubOpts). --spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid(), - emqx_types:subopts()) -> ok). -subscribe(Topic, SubId, Options) when is_atom(SubId); is_binary(SubId)-> - emqx_broker:subscribe(iolist_to_binary(Topic), SubId, Options); -subscribe(Topic, SubPid, Options) when is_pid(SubPid)-> - emqx_broker:subscribe(iolist_to_binary(Topic), SubPid, Options). +-spec(subscribe(emqx_topic:topic() | string(), + emqx_types:subid() | pid(), emqx_types:subopts()) -> ok). +subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubId, SubOpts). --spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}). +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). publish(Msg) -> emqx_broker:publish(Msg). @@ -91,26 +88,10 @@ publish(Msg) -> unsubscribe(Topic) -> emqx_broker:unsubscribe(iolist_to_binary(Topic)). --spec(unsubscribe(emqx_topic:topic() | string(), emqx_types:subid() | pid()) -> ok). -unsubscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubId); -unsubscribe(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:unsubscribe(iolist_to_binary(Topic), SubPid). - %%------------------------------------------------------------------------------ %% PubSub management API %%------------------------------------------------------------------------------ --spec(get_subopts(emqx_topic:topic() | string(), emqx_types:subscriber()) - -> emqx_types:subopts()). -get_subopts(Topic, Subscriber) -> - emqx_broker:get_subopts(iolist_to_binary(Topic), Subscriber). - --spec(set_subopts(emqx_topic:topic() | string(), emqx_types:subscriber(), - emqx_types:subopts()) -> boolean()). -set_subopts(Topic, Subscriber, Options) when is_map(Options) -> - emqx_broker:set_subopts(iolist_to_binary(Topic), Subscriber, Options). - -spec(topics() -> list(emqx_topic:topic())). topics() -> emqx_router:topics(). @@ -118,15 +99,15 @@ topics() -> emqx_router:topics(). subscribers(Topic) -> emqx_broker:subscribers(iolist_to_binary(Topic)). --spec(subscriptions(emqx_types:subscriber()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(Subscriber) -> - emqx_broker:subscriptions(Subscriber). +-spec(subscriptions(pid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). +subscriptions(SubPid) when is_pid(SubPid) -> + emqx_broker:subscriptions(SubPid). --spec(subscribed(emqx_topic:topic() | string(), pid() | emqx_types:subid()) -> boolean()). -subscribed(Topic, SubPid) when is_pid(SubPid) -> - emqx_broker:subscribed(iolist_to_binary(Topic), SubPid); -subscribed(Topic, SubId) when is_atom(SubId); is_binary(SubId) -> - emqx_broker:subscribed(iolist_to_binary(Topic), SubId). +-spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic() | string()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + emqx_broker:subscribed(SubPid, iolist_to_binary(Topic)); +subscribed(SubId, Topic) when is_atom(SubId); is_binary(SubId) -> + emqx_broker:subscribed(SubId, iolist_to_binary(Topic)). %%------------------------------------------------------------------------------ %% Hooks API diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 239769de6..1c43eb746 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -148,7 +148,7 @@ stop() -> %%----------------------------------------------------------------------------- init([]) -> - _ = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]), {ok, #{}}. handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) -> diff --git a/src/emqx_banned.erl b/src/emqx_banned.erl index aae4f8c7a..0c9ffdd10 100644 --- a/src/emqx_banned.erl +++ b/src/emqx_banned.erl @@ -26,17 +26,16 @@ -export([start_link/0]). -export([check/1]). --export([add/1, del/1]). +-export([add/1, delete/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(TAB, ?MODULE). --define(SERVER, ?MODULE). -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Mnesia bootstrap -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ mnesia(boot) -> ok = ekka_mnesia:create_table(?TAB, [ @@ -52,7 +51,7 @@ mnesia(copy) -> %% @doc Start the banned server. -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). -spec(check(emqx_types:credentials()) -> boolean()). check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) -> @@ -64,25 +63,25 @@ check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) - add(Banned) when is_record(Banned, banned) -> mnesia:dirty_write(?TAB, Banned). --spec(del({client_id, emqx_types:client_id()} | - {username, emqx_types:username()} | - {peername, emqx_types:peername()}) -> ok). -del(Key) -> +-spec(delete({client_id, emqx_types:client_id()} + | {username, emqx_types:username()} + | {peername, emqx_types:peername()}) -> ok). +delete(Key) -> mnesia:dirty_delete(?TAB, Key). -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> {ok, ensure_expiry_timer(#{expiry_timer => undefined})}. handle_call(Req, _From, State) -> - emqx_logger:error("[BANNED] unexpected call: ~p", [Req]), + emqx_logger:error("[Banned] unexpected call: ~p", [Req]), {reply, ignored, State}. handle_cast(Msg, State) -> - emqx_logger:error("[BANNED] unexpected msg: ~p", [Msg]), + emqx_logger:error("[Banned] unexpected msg: ~p", [Msg]), {noreply, State}. handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> @@ -90,7 +89,7 @@ handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> {noreply, ensure_expiry_timer(State), hibernate}; handle_info(Info, State) -> - emqx_logger:error("[BANNED] unexpected info: ~p", [Info]), + emqx_logger:error("[Banned] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> @@ -99,21 +98,22 @@ terminate(_Reason, #{expiry_timer := TRef}) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Internal functions -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ -ifdef(TEST). ensure_expiry_timer(State) -> State#{expiry_timer := emqx_misc:start_timer(timer:seconds(1), expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(5), expire)}. + State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> - mnesia:foldl(fun - (B = #banned{until = Until}, _Acc) when Until < Now -> - mnesia:delete_object(?TAB, B, sticky_write); - (_, _Acc) -> ok - end, ok, ?TAB). + mnesia:foldl( + fun(B = #banned{until = Until}, _Acc) when Until < Now -> + mnesia:delete_object(?TAB, B, sticky_write); + (_, _Acc) -> ok + end, ok, ?TAB). + diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 50fb06e0e..429c6097d 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -19,150 +19,163 @@ -include("emqx.hrl"). -export([start_link/2]). --export([subscribe/1, subscribe/2, subscribe/3, subscribe/4]). --export([multi_subscribe/1, multi_subscribe/2, multi_subscribe/3]). +-export([subscribe/1, subscribe/2, subscribe/3]). +-export([unsubscribe/1]). +-export([subscriber_down/1]). -export([publish/1, safe_publish/1]). --export([unsubscribe/1, unsubscribe/2, unsubscribe/3]). --export([multi_unsubscribe/1, multi_unsubscribe/2, multi_unsubscribe/3]). --export([dispatch/2, dispatch/3]). +-export([dispatch/2]). -export([subscriptions/1, subscribers/1, subscribed/2]). --export([get_subopts/2, set_subopts/3]). +-export([get_subopts/2, set_subopts/2]). -export([topics/0]). +%% Stats fun +-export([stats_fun/0]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-import(emqx_tables, [lookup_value/2, lookup_value/3]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. --record(state, {pool, id, submap, submon}). --record(subscribe, {topic, subpid, subid, subopts = #{}}). --record(unsubscribe, {topic, subpid, subid}). - -%% The default request timeout --define(TIMEOUT, 60000). -define(BROKER, ?MODULE). -%% ETS tables --define(SUBOPTION, emqx_suboption). --define(SUBSCRIBER, emqx_subscriber). +%% ETS tables for PubSub +-define(SUBOPTION, emqx_suboption). +-define(SUBSCRIBER, emqx_subscriber). -define(SUBSCRIPTION, emqx_subscription). +%% Guards -define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> - gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, - [Pool, Id], [{hibernate_after, 1000}]). + ok = create_tabs(), + gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, + ?MODULE, [Pool, Id], []). %%------------------------------------------------------------------------------ -%% Subscribe +%% Create tabs +%%------------------------------------------------------------------------------ + +-spec(create_tabs() -> ok). +create_tabs() -> + TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], + + %% SubOption: {SubPid, Topic} -> SubOption + ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]), + + %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... + %% duplicate_bag: o(1) insert + ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + + %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... + %% bag: o(n) insert:( + ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + +%%------------------------------------------------------------------------------ +%% Subscribe API %%------------------------------------------------------------------------------ -spec(subscribe(emqx_topic:topic()) -> ok). subscribe(Topic) when is_binary(Topic) -> - subscribe(Topic, self()). + subscribe(Topic, undefined). --spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok). -subscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - subscribe(Topic, SubPid, undefined); +-spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - subscribe(Topic, self(), SubId). + subscribe(Topic, SubId, #{qos => 0}); +subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> + subscribe(Topic, undefined, SubOpts). --spec(subscribe(emqx_topic:topic(), pid() | emqx_types:subid(), - emqx_types:subid() | emqx_types:subopts()) -> ok). -subscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - subscribe(Topic, SubPid, SubId, #{qos => 0}); -subscribe(Topic, SubPid, SubOpts) when is_binary(Topic), is_pid(SubPid), is_map(SubOpts) -> - subscribe(Topic, SubPid, undefined, SubOpts); +-spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts) -> - subscribe(Topic, self(), SubId, SubOpts). + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of + false -> + ok = emqx_broker_helper:register_sub(SubPid, SubId), + do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); + true -> ok + end. --spec(subscribe(emqx_topic:topic(), pid(), emqx_types:subid(), emqx_types:subopts()) -> ok). -subscribe(Topic, SubPid, SubId, SubOpts) when is_binary(Topic), is_pid(SubPid), - ?is_subid(SubId), is_map(SubOpts) -> - Broker = pick(SubPid), - SubReq = #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}, - wait_for_reply(async_call(Broker, SubReq), ?TIMEOUT). +with_subid(undefined, SubOpts) -> + SubOpts; +with_subid(SubId, SubOpts) -> + maps:put(subid, SubId, SubOpts). --spec(multi_subscribe(emqx_types:topic_table()) -> ok). -multi_subscribe(TopicTable) when is_list(TopicTable) -> - multi_subscribe(TopicTable, self()). +%% @private +do_subscribe(Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_subscribe(Group, Topic, SubPid, SubOpts). --spec(multi_subscribe(emqx_types:topic_table(), pid() | emqx_types:subid()) -> ok). -multi_subscribe(TopicTable, SubPid) when is_pid(SubPid) -> - multi_subscribe(TopicTable, SubPid, undefined); -multi_subscribe(TopicTable, SubId) when ?is_subid(SubId) -> - multi_subscribe(TopicTable, self(), SubId). +do_subscribe(undefined, Topic, SubPid, SubOpts) -> + case emqx_broker_helper:get_sub_shard(SubPid, Topic) of + 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + call(pick(Topic), {subscribe, Topic}); + I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, maps:put(shard, I, SubOpts)}), + call(pick({Topic, I}), {subscribe, Topic, I}) + end; --spec(multi_subscribe(emqx_types:topic_table(), pid(), emqx_types:subid()) -> ok). -multi_subscribe(TopicTable, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - SubReq = fun(Topic, SubOpts) -> - #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts} - end, - wait_for_replies([async_call(Broker, SubReq(Topic, SubOpts)) - || {Topic, SubOpts} <- TopicTable], ?TIMEOUT). +%% Shared subscription +do_subscribe(Group, Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + emqx_shared_sub:subscribe(Group, Topic, SubPid). %%------------------------------------------------------------------------------ -%% Unsubscribe +%% Unsubscribe API %%------------------------------------------------------------------------------ -spec(unsubscribe(emqx_topic:topic()) -> ok). unsubscribe(Topic) when is_binary(Topic) -> - unsubscribe(Topic, self()). + SubPid = self(), + case ets:lookup(?SUBOPTION, {SubPid, Topic}) of + [{_, SubOpts}] -> + _ = emqx_broker_helper:reclaim_seq(Topic), + do_unsubscribe(Topic, SubPid, SubOpts); + [] -> ok + end. --spec(unsubscribe(emqx_topic:topic(), pid() | emqx_types:subid()) -> ok). -unsubscribe(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - unsubscribe(Topic, SubPid, undefined); -unsubscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - unsubscribe(Topic, self(), SubId). +do_unsubscribe(Topic, SubPid, SubOpts) -> + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_unsubscribe(Group, Topic, SubPid, SubOpts). --spec(unsubscribe(emqx_topic:topic(), pid(), emqx_types:subid()) -> ok). -unsubscribe(Topic, SubPid, SubId) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - UnsubReq = #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}, - wait_for_reply(async_call(Broker, UnsubReq), ?TIMEOUT). +do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; --spec(multi_unsubscribe([emqx_topic:topic()]) -> ok). -multi_unsubscribe(Topics) -> - multi_unsubscribe(Topics, self()). - --spec(multi_unsubscribe([emqx_topic:topic()], pid() | emqx_types:subid()) -> ok). -multi_unsubscribe(Topics, SubPid) when is_pid(SubPid) -> - multi_unsubscribe(Topics, SubPid, undefined); -multi_unsubscribe(Topics, SubId) when ?is_subid(SubId) -> - multi_unsubscribe(Topics, self(), SubId). - --spec(multi_unsubscribe([emqx_topic:topic()], pid(), emqx_types:subid()) -> ok). -multi_unsubscribe(Topics, SubPid, SubId) when is_pid(SubPid), ?is_subid(SubId) -> - Broker = pick(SubPid), - UnsubReq = fun(Topic) -> - #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId} - end, - wait_for_replies([async_call(Broker, UnsubReq(Topic)) || Topic <- Topics], ?TIMEOUT). +do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> + emqx_shared_sub:unsubscribe(Group, Topic, SubPid). %%------------------------------------------------------------------------------ %% Publish %%------------------------------------------------------------------------------ --spec(publish(emqx_types:message()) -> {ok, emqx_types:deliver_results()}). +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). publish(Msg) when is_record(Msg, message) -> _ = emqx_tracer:trace(publish, Msg), - {ok, case emqx_hooks:run('message.publish', [], Msg) of - {ok, Msg1 = #message{topic = Topic}} -> - Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)), - Delivery#delivery.results; - {stop, _} -> - emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]), - [] - end}. + case emqx_hooks:run('message.publish', [], Msg) of + {ok, Msg1 = #message{topic = Topic}} -> + Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)), + Delivery#delivery.results; + {stop, _} -> + emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]), + [] + end. --spec(safe_publish(emqx_types:message()) -> ok). %% Called internally +-spec(safe_publish(emqx_types:message()) -> ok). safe_publish(Msg) when is_record(Msg, message) -> try publish(Msg) @@ -228,97 +241,137 @@ dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> inc_dropped_cnt(Topic), Delivery; [Sub] -> %% optimize? - dispatch(Sub, Topic, Msg), - Delivery#delivery{results = [{dispatch, Topic, 1}|Results]}; - Subscribers -> - Count = lists:foldl(fun(Sub, Acc) -> - dispatch(Sub, Topic, Msg), Acc + 1 - end, 0, Subscribers), - Delivery#delivery{results = [{dispatch, Topic, Count}|Results]} + Cnt = dispatch(Sub, Topic, Msg), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]}; + Subs -> + Cnt = lists:foldl( + fun(Sub, Acc) -> + dispatch(Sub, Topic, Msg) + Acc + end, 0, Subs), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]} end. -dispatch({SubPid, _SubId}, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; -dispatch({share, _Group, _Sub}, _Topic, _Msg) -> - ignored. +dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> + case erlang:is_process_alive(SubPid) of + true -> + SubPid ! {dispatch, Topic, Msg}, + 1; + false -> 0 + end; +dispatch({shard, I}, Topic, Msg) -> + lists:foldl( + fun(SubPid, Cnt) -> + dispatch(SubPid, Topic, Msg) + Cnt + end, 0, subscribers({shard, Topic, I})). inc_dropped_cnt(<<"$SYS/", _/binary>>) -> ok; inc_dropped_cnt(_Topic) -> emqx_metrics:inc('messages/dropped'). --spec(subscribers(emqx_topic:topic()) -> [emqx_types:subscriber()]). -subscribers(Topic) -> - try ets:lookup_element(?SUBSCRIBER, Topic, 2) catch error:badarg -> [] end. +-spec(subscribers(emqx_topic:topic()) -> [pid()]). +subscribers(Topic) when is_binary(Topic) -> + lookup_value(?SUBSCRIBER, Topic, []); +subscribers(Shard = {shard, _Topic, _I}) -> + lookup_value(?SUBSCRIBER, Shard, []). --spec(subscriptions(emqx_types:subscriber()) +%%------------------------------------------------------------------------------ +%% Subscriber is down +%%------------------------------------------------------------------------------ + +-spec(subscriber_down(pid()) -> true). +subscriber_down(SubPid) -> + lists:foreach( + fun(Topic) -> + case lookup_value(?SUBOPTION, {SubPid, Topic}) of + SubOpts when is_map(SubOpts) -> + _ = emqx_broker_helper:reclaim_seq(Topic), + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + undefined -> ok + end + end, lookup_value(?SUBSCRIPTION, SubPid, [])), + ets:delete(?SUBSCRIPTION, SubPid). + +%%------------------------------------------------------------------------------ +%% Management APIs +%%------------------------------------------------------------------------------ + +-spec(subscriptions(pid() | emqx_types:subid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). -subscriptions(Subscriber) -> - lists:map(fun({_, {share, _Group, Topic}}) -> - subscription(Topic, Subscriber); - ({_, Topic}) -> - subscription(Topic, Subscriber) - end, ets:lookup(?SUBSCRIPTION, Subscriber)). - -subscription(Topic, Subscriber) -> - {Topic, ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2)}. - --spec(subscribed(emqx_topic:topic(), pid() | emqx_types:subid() | emqx_types:subscriber()) -> boolean()). -subscribed(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - case ets:match_object(?SUBOPTION, {{Topic, {SubPid, '_'}}, '_'}, 1) of - {Match, _} -> - length(Match) >= 1; - '$end_of_table' -> - false - end; -subscribed(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> - case ets:match_object(?SUBOPTION, {{Topic, {'_', SubId}}, '_'}, 1) of - {Match, _} -> - length(Match) >= 1; - '$end_of_table' -> - false - end; -subscribed(Topic, {SubPid, SubId}) when is_binary(Topic), is_pid(SubPid), ?is_subid(SubId) -> - ets:member(?SUBOPTION, {Topic, {SubPid, SubId}}). - --spec(get_subopts(emqx_topic:topic(), emqx_types:subscriber()) -> emqx_types:subopts()). -get_subopts(Topic, Subscriber) when is_binary(Topic) -> - try ets:lookup_element(?SUBOPTION, {Topic, Subscriber}, 2) - catch error:badarg -> [] +subscriptions(SubPid) when is_pid(SubPid) -> + [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})} + || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])]; +subscriptions(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + subscriptions(SubPid); + undefined -> [] end. --spec(set_subopts(emqx_topic:topic(), emqx_types:subscriber(), emqx_types:subopts()) -> boolean()). -set_subopts(Topic, Subscriber, Opts) when is_binary(Topic), is_map(Opts) -> - case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of +-spec(subscribed(pid(), emqx_topic:topic()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + ets:member(?SUBOPTION, {SubPid, Topic}); +subscribed(SubId, Topic) when ?is_subid(SubId) -> + SubPid = emqx_broker_helper:lookup_subpid(SubId), + ets:member(?SUBOPTION, {SubPid, Topic}). + +-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts() | undefined). +get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> + lookup_value(?SUBOPTION, {SubPid, Topic}); +get_subopts(SubId, Topic) when ?is_subid(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + get_subopts(SubPid, Topic); + undefined -> undefined + end. + +-spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). +set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> + Sub = {self(), Topic}, + case ets:lookup(?SUBOPTION, Sub) of [{_, OldOpts}] -> - ets:insert(?SUBOPTION, {{Topic, Subscriber}, maps:merge(OldOpts, Opts)}); + ets:insert(?SUBOPTION, {Sub, maps:merge(OldOpts, NewOpts)}); [] -> false end. -async_call(Broker, Req) -> - From = {self(), Tag = make_ref()}, - ok = gen_server:cast(Broker, {From, Req}), - Tag. +-spec(topics() -> [emqx_topic:topic()]). +topics() -> + emqx_router:topics(). -wait_for_replies(Tags, Timeout) -> - lists:foreach( - fun(Tag) -> - wait_for_reply(Tag, Timeout) - end, Tags). +%%------------------------------------------------------------------------------ +%% Stats fun +%%------------------------------------------------------------------------------ -wait_for_reply(Tag, Timeout) -> - receive - {Tag, Reply} -> Reply - after Timeout -> - exit(timeout) +stats_fun() -> + safe_update_stats(?SUBSCRIBER, 'subscribers/count', 'subscribers/max'), + safe_update_stats(?SUBSCRIPTION, 'subscriptions/count', 'subscriptions/max'), + safe_update_stats(?SUBOPTION, 'suboptions/count', 'suboptions/max'). + +safe_update_stats(Tab, Stat, MaxStat) -> + case ets:info(Tab, size) of + undefined -> ok; + Size -> emqx_stats:setstat(Stat, MaxStat, Size) end. -%% Pick a broker -pick(SubPid) when is_pid(SubPid) -> - gproc_pool:pick_worker(broker, SubPid). +%%------------------------------------------------------------------------------ +%% call, cast, pick +%%------------------------------------------------------------------------------ --spec(topics() -> [emqx_topic:topic()]). -topics() -> emqx_router:topics(). +call(Broker, Req) -> + gen_server:call(Broker, Req). + +cast(Broker, Msg) -> + gen_server:cast(Broker, Msg). + +%% Pick a broker +pick(Topic) -> + gproc_pool:pick_worker(broker_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks @@ -326,38 +379,49 @@ topics() -> emqx_router:topics(). init([Pool, Id]) -> true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, #state{pool = Pool, id = Id, submap = #{}, submon = emqx_pmon:new()}}. + {ok, #{pool => Pool, id => Id}}. + +handle_call({subscribe, Topic}, _From, State) -> + Ok = emqx_router:do_add_route(Topic), + {reply, Ok, State}; + +handle_call({subscribe, Topic, I}, _From, State) -> + Ok = case get(Shard = {Topic, I}) of + undefined -> + _ = put(Shard, true), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {subscribe, Topic}); + true -> ok + end, + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Broker] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({From, #subscribe{topic = Topic, subpid = SubPid, subid = SubId, subopts = SubOpts}}, State) -> - Subscriber = {SubPid, SubId}, - case ets:member(?SUBOPTION, {Topic, Subscriber}) of - false -> - Group = maps:get(share, SubOpts, undefined), - true = do_subscribe(Group, Topic, Subscriber, SubOpts), - emqx_shared_sub:subscribe(Group, Topic, SubPid), - emqx_router:add_route(From, Topic, dest(Group)), - {noreply, monitor_subscriber(Subscriber, State)}; - true -> - gen_server:reply(From, ok), - {noreply, State} - end; +handle_cast({subscribe, Topic}, State) -> + case emqx_router:do_add_route(Topic) of + ok -> ok; + {error, Reason} -> + emqx_logger:error("[Broker] Failed to add route: ~p", [Reason]) + end, + {noreply, State}; -handle_cast({From, #unsubscribe{topic = Topic, subpid = SubPid, subid = SubId}}, State) -> - Subscriber = {SubPid, SubId}, - case ets:lookup(?SUBOPTION, {Topic, Subscriber}) of - [{_, SubOpts}] -> - Group = maps:get(share, SubOpts, undefined), - true = do_unsubscribe(Group, Topic, Subscriber), - emqx_shared_sub:unsubscribe(Group, Topic, SubPid), - case ets:member(?SUBSCRIBER, Topic) of - false -> emqx_router:del_route(From, Topic, dest(Group)); - true -> gen_server:reply(From, ok) - end; - [] -> gen_server:reply(From, ok) +handle_cast({unsubscribed, Topic}, State) -> + case ets:member(?SUBSCRIBER, Topic) of + false -> + _ = emqx_router:do_delete_route(Topic); + true -> ok + end, + {noreply, State}; + +handle_cast({unsubscribed, Topic, I}, State) -> + case ets:member(?SUBSCRIBER, {shard, Topic, I}) of + false -> + _ = erase({Topic, I}), + true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {unsubscribed, Topic}); + true -> ok end, {noreply, State}; @@ -365,21 +429,11 @@ handle_cast(Msg, State) -> emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #state{submap = SubMap}) -> - case maps:find(SubPid, SubMap) of - {ok, SubIds} -> - lists:foreach(fun(SubId) -> subscriber_down({SubPid, SubId}) end, SubIds), - {noreply, demonitor_subscriber(SubPid, State)}; - error -> - emqx_logger:error("unexpected 'DOWN': ~p, reason: ~p", [SubPid, Reason]), - {noreply, State} - end; - handle_info(Info, State) -> emqx_logger:error("[Broker] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{pool = Pool, id = Id}) -> +terminate(_Reason, #{pool := Pool, id := Id}) -> gproc_pool:disconnect_worker(Pool, {Pool, Id}). code_change(_OldVsn, State, _Extra) -> @@ -389,52 +443,3 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -do_subscribe(Group, Topic, Subscriber, SubOpts) -> - ets:insert(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}), - ets:insert(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}), - ets:insert(?SUBOPTION, {{Topic, Subscriber}, SubOpts}). - -do_unsubscribe(Group, Topic, Subscriber) -> - ets:delete_object(?SUBSCRIPTION, {Subscriber, shared(Group, Topic)}), - ets:delete_object(?SUBSCRIBER, {Topic, shared(Group, Subscriber)}), - ets:delete(?SUBOPTION, {Topic, Subscriber}). - -subscriber_down(Subscriber) -> - Topics = lists:map(fun({_, {share, Group, Topic}}) -> - {Topic, Group}; - ({_, Topic}) -> - {Topic, undefined} - end, ets:lookup(?SUBSCRIPTION, Subscriber)), - lists:foreach(fun({Topic, undefined}) -> - true = do_unsubscribe(undefined, Topic, Subscriber), - ets:member(?SUBSCRIBER, Topic) orelse emqx_router:del_route(Topic, dest(undefined)); - ({Topic, Group}) -> - true = do_unsubscribe(Group, Topic, Subscriber), - Groups = groups(Topic), - case lists:member(Group, lists:usort(Groups)) of - true -> ok; - false -> emqx_router:del_route(Topic, dest(Group)) - end - end, Topics). - -monitor_subscriber({SubPid, SubId}, State = #state{submap = SubMap, submon = SubMon}) -> - UpFun = fun(SubIds) -> lists:usort([SubId|SubIds]) end, - State#state{submap = maps:update_with(SubPid, UpFun, [SubId], SubMap), - submon = emqx_pmon:monitor(SubPid, SubMon)}. - -demonitor_subscriber(SubPid, State = #state{submap = SubMap, submon = SubMon}) -> - State#state{submap = maps:remove(SubPid, SubMap), - submon = emqx_pmon:demonitor(SubPid, SubMon)}. - -dest(undefined) -> node(); -dest(Group) -> {Group, node()}. - -shared(undefined, Name) -> Name; -shared(Group, Name) -> {share, Group, Name}. - -groups(Topic) -> - lists:foldl(fun({_, {share, Group, _}}, Acc) -> - [Group | Acc]; - ({_, _}, Acc) -> - Acc - end, [], ets:lookup(?SUBSCRIBER, Topic)). diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index e597a233e..1ea3f3668 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -17,42 +17,110 @@ -behaviour(gen_server). -export([start_link/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([register_sub/2]). +-export([lookup_subid/1, lookup_subpid/1]). +-export([get_sub_shard/2]). +-export([create_seq/1, reclaim_seq/1]). -%% internal export --export([stats_fun/0]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). -define(HELPER, ?MODULE). +-define(SUBID, emqx_subid). +-define(SUBMON, emqx_submon). +-define(SUBSEQ, emqx_subseq). +-define(SHARD, 1024). --record(state, {}). +-define(BATCH_SIZE, 100000). --spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). +-spec(register_sub(pid(), emqx_types:subid()) -> ok). +register_sub(SubPid, SubId) when is_pid(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [] -> + gen_server:cast(?HELPER, {register_sub, SubPid, SubId}); + [{_, SubId}] -> + ok; + _Other -> + error(subid_conflict) + end. + +-spec(lookup_subid(pid()) -> emqx_types:subid() | undefined). +lookup_subid(SubPid) when is_pid(SubPid) -> + emqx_tables:lookup_value(?SUBMON, SubPid). + +-spec(lookup_subpid(emqx_types:subid()) -> pid()). +lookup_subpid(SubId) -> + emqx_tables:lookup_value(?SUBID, SubId). + +-spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_sub_shard(SubPid, Topic) -> + case create_seq(Topic) of + Seq when Seq =< ?SHARD -> 0; + _ -> erlang:phash2(SubPid, shards_num()) + 1 + end. + +-spec(shards_num() -> pos_integer()). +shards_num() -> + %% Dynamic sharding later... + ets:lookup_element(?HELPER, shards, 2). + +-spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +create_seq(Topic) -> + emqx_sequence:nextval(?SUBSEQ, Topic). + +-spec(reclaim_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +reclaim_seq(Topic) -> + emqx_sequence:reclaim(?SUBSEQ, Topic). + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ init([]) -> - %% Use M:F/A for callback, not anonymous function because - %% fun M:F/A is small, also no badfun risk during hot beam reload - emqx_stats:update_interval(broker_stats, fun ?MODULE:stats_fun/0), - {ok, #state{}, hibernate}. + %% Helper table + ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + %% Shards: CPU * 32 + true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), + %% SubSeq: Topic -> SeqId + ok = emqx_sequence:create(?SUBSEQ), + %% SubId: SubId -> SubPid + ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + %% SubMon: SubPid -> SubId + ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + %% Stats timer + ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), + {ok, #{pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]), - {reply, ignored, State}. + {reply, ignored, State}. + +handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> + true = (SubId =:= undefined) orelse ets:insert(?SUBID, {SubId, SubPid}), + true = ets:insert(?SUBMON, {SubPid, SubId}), + {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), {noreply, State}. +handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> + SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, SubPids]), + {_, PMon1} = emqx_pmon:erase_all(SubPids, PMon), + {noreply, State#{pmon := PMon1}}; + handle_info(Info, State) -> emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> +terminate(_Reason, _State) -> + true = emqx_sequence:delete(?SUBSEQ), emqx_stats:cancel_update(broker_stats). code_change(_OldVsn, State, _Extra) -> @@ -62,17 +130,13 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -stats_fun() -> - safe_update_stats(emqx_subscriber, - 'subscribers/count', 'subscribers/max'), - safe_update_stats(emqx_subscription, - 'subscriptions/count', 'subscriptions/max'), - safe_update_stats(emqx_suboptions, - 'suboptions/count', 'suboptions/max'). - -safe_update_stats(Tab, Stat, MaxStat) -> - case ets:info(Tab, size) of - undefined -> ok; - Size -> emqx_stats:setstat(Stat, MaxStat, Size) +clean_down(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [{_, SubId}] -> + true = ets:delete(?SUBMON, SubPid), + true = (SubId =:= undefined) + orelse ets:delete_object(?SUBID, {SubId, SubPid}), + emqx_broker:subscriber_down(SubPid); + [] -> ok end. diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl index 51f6e72aa..5b1c0a0e7 100644 --- a/src/emqx_broker_sup.erl +++ b/src/emqx_broker_sup.erl @@ -20,8 +20,6 @@ -export([init/1]). --define(TAB_OPTS, [public, {read_concurrency, true}, {write_concurrency, true}]). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -30,39 +28,26 @@ start_link() -> %%------------------------------------------------------------------------------ init([]) -> - %% Create the pubsub tables - ok = lists:foreach(fun create_tab/1, [subscription, subscriber, suboption]), - - %% Shared subscription - SharedSub = {shared_sub, {emqx_shared_sub, start_link, []}, - permanent, 5000, worker, [emqx_shared_sub]}, - - %% Broker helper - Helper = {broker_helper, {emqx_broker_helper, start_link, []}, - permanent, 5000, worker, [emqx_broker_helper]}, - %% Broker pool - BrokerPool = emqx_pool_sup:spec(emqx_broker_pool, - [broker, hash, emqx_vm:schedulers() * 2, + PoolSize = emqx_vm:schedulers() * 2, + BrokerPool = emqx_pool_sup:spec([broker_pool, hash, PoolSize, {emqx_broker, start_link, []}]), - {ok, {{one_for_all, 0, 1}, [SharedSub, Helper, BrokerPool]}}. + %% Shared subscription + SharedSub = #{id => shared_sub, + start => {emqx_shared_sub, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_shared_sub]}, -%%------------------------------------------------------------------------------ -%% Create tables -%%------------------------------------------------------------------------------ + %% Broker helper + Helper = #{id => helper, + start => {emqx_broker_helper, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_broker_helper]}, -create_tab(suboption) -> - %% Suboption: {Topic, Sub} -> [{qos, 1}] - emqx_tables:new(emqx_suboption, [set | ?TAB_OPTS]); - -create_tab(subscriber) -> - %% Subscriber: Topic -> Sub1, Sub2, Sub3, ..., SubN - %% duplicate_bag: o(1) insert - emqx_tables:new(emqx_subscriber, [duplicate_bag | ?TAB_OPTS]); - -create_tab(subscription) -> - %% Subscription: Sub -> Topic1, Topic2, Topic3, ..., TopicN - %% bag: o(n) insert - emqx_tables:new(emqx_subscription, [bag | ?TAB_OPTS]). + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. diff --git a/src/emqx_cli.erl b/src/emqx_cli.erl index 6be9093b5..f7d513e9d 100644 --- a/src/emqx_cli.erl +++ b/src/emqx_cli.erl @@ -17,16 +17,17 @@ -export([print/1, print/2, usage/1, usage/2]). print(Msg) -> - io:format(Msg). + io:format(Msg), lists:flatten(io_lib:format("~p", [Msg])). print(Format, Args) -> - io:format(Format, Args). + io:format(Format, Args), lists:flatten(io_lib:format(Format, Args)). usage(CmdList) -> - lists:foreach( + lists:map( fun({Cmd, Descr}) -> - io:format("~-48s# ~s~n", [Cmd, Descr]) + io:format("~-48s# ~s~n", [Cmd, Descr]), + lists:flatten(io_lib:format("~-48s# ~s~n", [Cmd, Descr])) end, CmdList). usage(Format, Args) -> - usage([{Format, Args}]). \ No newline at end of file + usage([{Format, Args}]). diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 19892b386..b8a57bc50 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -20,102 +20,110 @@ -export([start_link/0]). --export([lookup_connection/1]). -export([register_connection/1, register_connection/2]). --export([unregister_connection/1]). --export([get_conn_attrs/1, set_conn_attrs/2]). --export([get_conn_stats/1, set_conn_stats/2]). +-export([unregister_connection/1, unregister_connection/2]). +-export([get_conn_attrs/1, get_conn_attrs/2]). +-export([set_conn_attrs/2, set_conn_attrs/3]). +-export([get_conn_stats/1, get_conn_stats/2]). +-export([set_conn_stats/2, set_conn_stats/3]). -export([lookup_conn_pid/1]). +%% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). %% internal export --export([update_conn_stats/0]). +-export([stats_fun/0]). -define(CM, ?MODULE). -%% ETS Tables. --define(CONN_TAB, emqx_conn). +%% ETS tables for connection management. +-define(CONN_TAB, emqx_conn). -define(CONN_ATTRS_TAB, emqx_conn_attrs). -define(CONN_STATS_TAB, emqx_conn_stats). +-define(BATCH_SIZE, 100000). + %% @doc Start the connection manager. -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?CM}, ?MODULE, [], []). -%% @doc Lookup a connection. --spec(lookup_connection(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). -lookup_connection(ClientId) when is_binary(ClientId) -> - ets:lookup(?CONN_TAB, ClientId). +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ %% @doc Register a connection. --spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). +-spec(register_connection(emqx_types:client_id()) -> ok). register_connection(ClientId) when is_binary(ClientId) -> - register_connection({ClientId, self()}); + register_connection(ClientId, self()). -register_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:insert(?CONN_TAB, Conn), +-spec(register_connection(emqx_types:client_id(), pid()) -> ok). +register_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = ets:insert(?CONN_TAB, {ClientId, ConnPid}), notify({registered, ClientId, ConnPid}). --spec(register_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}, list()) -> ok). -register_connection(ClientId, Attrs) when is_binary(ClientId) -> - register_connection({ClientId, self()}, Attrs); -register_connection(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> - set_conn_attrs(Conn, Attrs), - register_connection(Conn). +%% @doc Unregister a connection. +-spec(unregister_connection(emqx_types:client_id()) -> ok). +unregister_connection(ClientId) when is_binary(ClientId) -> + unregister_connection(ClientId, self()). + +-spec(unregister_connection(emqx_types:client_id(), pid()) -> ok). +unregister_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = do_unregister_connection({ClientId, ConnPid}), + notify({unregistered, ConnPid}). + +do_unregister_connection(Conn) -> + true = ets:delete(?CONN_STATS_TAB, Conn), + true = ets:delete(?CONN_ATTRS_TAB, Conn), + true = ets:delete_object(?CONN_TAB, Conn). %% @doc Get conn attrs --spec(get_conn_attrs({emqx_types:client_id(), pid()}) -> list()). -get_conn_attrs(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - try - ets:lookup_element(?CONN_ATTRS_TAB, Conn, 2) - catch - error:badarg -> [] - end. +-spec(get_conn_attrs(emqx_types:client_id()) -> list()). +get_conn_attrs(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_attrs(ClientId, ConnPid). + +-spec(get_conn_attrs(emqx_types:client_id(), pid()) -> list()). +get_conn_attrs(ClientId, ConnPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_ATTRS_TAB, {ClientId, ConnPid}, []). %% @doc Set conn attrs +-spec(set_conn_attrs(emqx_types:client_id(), list()) -> true). set_conn_attrs(ClientId, Attrs) when is_binary(ClientId) -> - set_conn_attrs({ClientId, self()}, Attrs); -set_conn_attrs(Conn = {ClientId, ConnPid}, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> + set_conn_attrs(ClientId, self(), Attrs). + +-spec(set_conn_attrs(emqx_types:client_id(), pid(), list()) -> true). +set_conn_attrs(ClientId, ConnPid, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, ets:insert(?CONN_ATTRS_TAB, {Conn, Attrs}). -%% @doc Unregister a conn. --spec(unregister_connection(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). -unregister_connection(ClientId) when is_binary(ClientId) -> - unregister_connection({ClientId, self()}); - -unregister_connection(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - _ = ets:delete(?CONN_STATS_TAB, Conn), - _ = ets:delete(?CONN_ATTRS_TAB, Conn), - _ = ets:delete_object(?CONN_TAB, Conn), - notify({unregistered, ClientId, ConnPid}). - -%% @doc Lookup connection pid --spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). -lookup_conn_pid(ClientId) when is_binary(ClientId) -> - case ets:lookup(?CONN_TAB, ClientId) of - [] -> undefined; - [{_, Pid}] -> Pid - end. - %% @doc Get conn stats --spec(get_conn_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). -get_conn_stats(Conn = {ClientId, ConnPid}) when is_binary(ClientId), is_pid(ConnPid) -> - try ets:lookup_element(?CONN_STATS_TAB, Conn, 2) - catch - error:badarg -> [] - end. +-spec(get_conn_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_stats(ClientId, ConnPid). + +-spec(get_conn_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId, ConnPid) when is_binary(ClientId) -> + Conn = {ClientId, ConnPid}, + emqx_tables:lookup_value(?CONN_STATS_TAB, Conn, []). %% @doc Set conn stats. --spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> boolean()). +-spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> true). set_conn_stats(ClientId, Stats) when is_binary(ClientId) -> - set_conn_stats({ClientId, self()}, Stats); + set_conn_stats(ClientId, self(), Stats). -set_conn_stats(Conn = {ClientId, ConnPid}, Stats) when is_binary(ClientId), is_pid(ConnPid) -> +-spec(set_conn_stats(emqx_types:client_id(), pid(), list(emqx_stats:stats())) -> true). +set_conn_stats(ClientId, ConnPid, Stats) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, ets:insert(?CONN_STATS_TAB, {Conn, Stats}). +%% @doc Lookup connection pid. +-spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). +lookup_conn_pid(ClientId) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_TAB, ClientId). + notify(Msg) -> gen_server:cast(?CM, {notify, Msg}). @@ -125,10 +133,10 @@ notify(Msg) -> init([]) -> TabOpts = [public, set, {write_concurrency, true}], - _ = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), - _ = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), - _ = emqx_tables:new(?CONN_STATS_TAB, TabOpts), - ok = emqx_stats:update_interval(cm_stats, fun ?MODULE:update_conn_stats/0), + ok = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?CONN_STATS_TAB, TabOpts), + ok = emqx_stats:update_interval(conn_stats, fun ?MODULE:stats_fun/0), {ok, #{conn_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> @@ -138,28 +146,26 @@ handle_call(Req, _From, State) -> handle_cast({notify, {registered, ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:monitor(ConnPid, ClientId, PMon)}}; -handle_cast({notify, {unregistered, _ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> +handle_cast({notify, {unregistered, ConnPid}}, State = #{conn_pmon := PMon}) -> {noreply, State#{conn_pmon := emqx_pmon:demonitor(ConnPid, PMon)}}; handle_cast(Msg, State) -> emqx_logger:error("[CM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, ConnPid, _Reason}, State = #{conn_pmon := PMon}) -> - case emqx_pmon:find(ConnPid, PMon) of - undefined -> - {noreply, State}; - ClientId -> - unregister_connection({ClientId, ConnPid}), - {noreply, State#{conn_pmon := emqx_pmon:erase(ConnPid, PMon)}} - end; +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{conn_pmon := PMon}) -> + ConnPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ConnPids, PMon), + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{conn_pmon := PMon1}}; handle_info(Info, State) -> emqx_logger:error("[CM] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(cm_stats). + emqx_stats:cancel_update(conn_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -168,7 +174,16 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -update_conn_stats() -> +clean_down({Pid, ClientId}) -> + Conn = {ClientId, Pid}, + case ets:member(?CONN_TAB, ClientId) + orelse ets:member(?CONN_ATTRS_TAB, Conn) of + true -> + do_unregister_connection(Conn); + false -> false + end. + +stats_fun() -> case ets:info(?CONN_TAB, size) of undefined -> ok; Size -> emqx_stats:setstat('connections/count', 'connections/max', Size) diff --git a/src/emqx_cm_sup.erl b/src/emqx_cm_sup.erl index 000e79336..e8c99c16b 100644 --- a/src/emqx_cm_sup.erl +++ b/src/emqx_cm_sup.erl @@ -1,6 +1,5 @@ %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% -%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -25,17 +24,17 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Banned = #{id => banned, - start => {emqx_banned, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_banned]}, - Manager = #{id => manager, - start => {emqx_cm, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm]}, + Banned = #{id => banned, + start => {emqx_banned, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [emqx_banned]}, + Manager = #{id => manager, + start => {emqx_cm, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_cm]}, {ok, {{one_for_one, 10, 100}, [Banned, Manager]}}. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 3d0ab0df7..9f2572b32 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -16,15 +16,14 @@ -behaviour(gen_server). --define(LOG_HEADER, "[TCP]"). - -include("emqx.hrl"). -include("emqx_mqtt.hrl"). + +-define(LOG_HEADER, "[MQTT]"). -include("logger.hrl"). -export([start_link/3]). --export([info/1, attrs/1]). --export([stats/1]). +-export([info/1, attrs/1, stats/1]). -export([kick/1]). -export([session/1]). @@ -38,19 +37,20 @@ peername, sockname, conn_state, - await_recv, + active_n, proto_state, parser_state, + gc_state, keepalive, enable_stats, stats_timer, - incoming, rate_limit, - publish_limit, + pub_limit, limit_timer, idle_timeout }). +-define(DEFAULT_ACTIVE_N, 100). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). start_link(Transport, Socket, Options) -> @@ -64,22 +64,22 @@ start_link(Transport, Socket, Options) -> info(CPid) when is_pid(CPid) -> call(CPid, info); -info(#state{transport = Transport, - socket = Socket, - peername = Peername, - sockname = Sockname, - conn_state = ConnState, - await_recv = AwaitRecv, - rate_limit = RateLimit, - publish_limit = PubLimit, - proto_state = ProtoState}) -> +info(#state{transport = Transport, + socket = Socket, + peername = Peername, + sockname = Sockname, + conn_state = ConnState, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState}) -> ConnInfo = [{socktype, Transport:type(Socket)}, {peername, Peername}, {sockname, Sockname}, {conn_state, ConnState}, - {await_recv, AwaitRecv}, + {active_n, ActiveN}, {rate_limit, esockd_rate_limit:info(RateLimit)}, - {publish_limit, esockd_rate_limit:info(PubLimit)}], + {pub_limit, esockd_rate_limit:info(PubLimit)}], ProtoInfo = emqx_protocol:info(ProtoState), lists:usort(lists:append(ConnInfo, ProtoInfo)). @@ -87,8 +87,8 @@ info(#state{transport = Transport, attrs(CPid) when is_pid(CPid) -> call(CPid, attrs); -attrs(#state{peername = Peername, - sockname = Sockname, +attrs(#state{peername = Peername, + sockname = Sockname, proto_state = ProtoState}) -> SockAttrs = [{peername, Peername}, {sockname, Sockname}], @@ -129,6 +129,7 @@ init([Transport, RawSocket, Options]) -> Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), RateLimit = init_limiter(proplists:get_value(rate_limit, Options)), PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)), + ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N), EnableStats = emqx_zone:get_env(Zone, enable_stats, true), IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), SendFun = send_fun(Transport, Socket), @@ -137,22 +138,22 @@ init([Transport, RawSocket, Options]) -> peercert => Peercert, sendfun => SendFun}, Options), ParserState = emqx_protocol:parser(ProtoState), - State = run_socket(#state{transport = Transport, - socket = Socket, - peername = Peername, - await_recv = false, - conn_state = running, - rate_limit = RateLimit, - publish_limit = PubLimit, - proto_state = ProtoState, - parser_state = ParserState, - enable_stats = EnableStats, - idle_timeout = IdleTimout - }), GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), - ok = emqx_gc:init(GcPolicy), + GcState = emqx_gc:init(GcPolicy), + State = run_socket(#state{transport = Transport, + socket = Socket, + peername = Peername, + conn_state = running, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState, + parser_state = ParserState, + gc_state = GcState, + enable_stats = EnableStats, + idle_timeout = IdleTimout + }), ok = emqx_misc:init_proc_mng_policy(Zone), - emqx_logger:set_metadata_peername(esockd_net:format(Peername)), gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State, self(), IdleTimout); @@ -205,16 +206,16 @@ handle_cast(Msg, State) -> handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> case emqx_protocol:deliver(PubOrAck, ProtoState) of {ok, ProtoState1} -> - State1 = ensure_stats_timer(State#state{proto_state = ProtoState1}), - ok = maybe_gc(State1, PubOrAck), - {noreply, State1}; + State1 = State#state{proto_state = ProtoState1}, + {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))}; {error, Reason} -> shutdown(Reason, State) end; + handle_info({timeout, Timer, emit_stats}, State = #state{stats_timer = Timer, - proto_state = ProtoState - }) -> + proto_state = ProtoState, + gc_state = GcState}) -> emqx_metrics:commit(), emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), NewState = State#state{stats_timer = undefined}, @@ -223,12 +224,14 @@ handle_info({timeout, Timer, emit_stats}, continue -> {noreply, NewState}; hibernate -> - ok = emqx_gc:reset(), - {noreply, NewState, hibernate}; + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; {shutdown, Reason} -> ?LOG(warning, "shutdown due to ~p", [Reason]), shutdown(Reason, NewState) end; + handle_info(timeout, State) -> shutdown(idle_timeout, State); @@ -243,19 +246,26 @@ handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), shutdown(conflict, State); +handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> + process_incoming(Data, State); + +%% Rate limit here, cool:) +handle_info({tcp_passive, _Sock}, State) -> + {noreply, run_socket(ensure_rate_limit(State))}; +%% FIXME Later +handle_info({ssl_passive, _Sock}, State) -> + {noreply, run_socket(ensure_rate_limit(State))}; + +handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error -> + shutdown(Reason, State); + +handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed -> + shutdown(closed, State); + +%% Rate limit timer handle_info(activate_sock, State) -> {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; -handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> - ?LOG(debug, "RECV ~p", [Data]), - Size = iolist_size(Data), - emqx_metrics:trans(inc, 'bytes/received', Size), - Incoming = #{bytes => Size, packets => 0}, - handle_packet(Data, State#state{await_recv = false, incoming = Incoming}); - -handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> - shutdown(Reason, State); - handle_info({inet_reply, _Sock, ok}, State) -> {noreply, State}; @@ -310,26 +320,37 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. %%------------------------------------------------------------------------------ -%% Parse and handle packets +%% Internals: process incoming, parse and handle packets %%------------------------------------------------------------------------------ -%% Receive and parse data -handle_packet(<<>>, State0) -> - State = ensure_stats_timer(ensure_rate_limit(State0)), - ok = maybe_gc(State, incoming), +process_incoming(Data, State) -> + Oct = iolist_size(Data), + ?LOG(debug, "RECV ~p", [Data]), + emqx_pd:update_counter(incoming_bytes, Oct), + emqx_metrics:trans(inc, 'bytes/received', Oct), + case handle_packet(Data, State) of + {noreply, State1} -> + State2 = maybe_gc({1, Oct}, State1), + {noreply, ensure_stats_timer(State2)}; + Shutdown -> Shutdown + end. + +%% Parse and handle packets +handle_packet(<<>>, State) -> {noreply, State}; + handle_packet(Data, State = #state{proto_state = ProtoState, parser_state = ParserState, idle_timeout = IdleTimeout}) -> - case catch emqx_frame:parse(Data, ParserState) of - {more, NewParserState} -> - {noreply, run_socket(State#state{parser_state = NewParserState}), IdleTimeout}; + try emqx_frame:parse(Data, ParserState) of + {more, ParserState1} -> + {noreply, State#state{parser_state = ParserState1}, IdleTimeout}; {ok, Packet = ?PACKET(Type), Rest} -> emqx_metrics:received(Packet), + (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1), case emqx_protocol:received(Packet, ProtoState) of {ok, ProtoState1} -> - NewState = State#state{proto_state = ProtoState1}, - handle_packet(Rest, inc_publish_cnt(Type, reset_parser(NewState))); + handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1})); {error, Reason} -> ?LOG(error, "Process packet error - ~p", [Reason]), shutdown(Reason, State); @@ -338,38 +359,32 @@ handle_packet(Data, State = #state{proto_state = ProtoState, {stop, Error, ProtoState1} -> stop(Error, State#state{proto_state = ProtoState1}) end; - {error, Error} -> - ?LOG(error, "Framing error - ~p", [Error]), - shutdown(Error, State); - {'EXIT', Reason} -> - ?LOG(error, "Parse failed for ~p~nError data:~p", [Reason, Data]), + {error, Reason} -> + ?LOG(error, "Parse frame error - ~p", [Reason]), + shutdown(Reason, State) + catch + _:Error -> + ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]), shutdown(parse_error, State) end. reset_parser(State = #state{proto_state = ProtoState}) -> State#state{parser_state = emqx_protocol:parser(ProtoState)}. -inc_publish_cnt(Type, State = #state{incoming = Incoming = #{packets := Cnt}}) - when Type == ?PUBLISH; Type == ?SUBSCRIBE -> - State#state{incoming = Incoming#{packets := Cnt + 1}}; -inc_publish_cnt(_Type, State) -> - State. - %%------------------------------------------------------------------------------ %% Ensure rate limit -%%------------------------------------------------------------------------------ -ensure_rate_limit(State = #state{rate_limit = Rl, publish_limit = Pl, - incoming = #{packets := Packets, bytes := Bytes}}) -> - ensure_rate_limit([{Pl, #state.publish_limit, Packets}, - {Rl, #state.rate_limit, Bytes}], State). +ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl}) -> + Limiters = [{Pl, #state.pub_limit, emqx_pd:reset_counter(incoming_pubs)}, + {Rl, #state.rate_limit, emqx_pd:reset_counter(incoming_bytes)}], + ensure_rate_limit(Limiters, State). ensure_rate_limit([], State) -> - run_socket(State); -ensure_rate_limit([{undefined, _Pos, _Num}|Limiters], State) -> + State; +ensure_rate_limit([{undefined, _Pos, _Cnt}|Limiters], State) -> ensure_rate_limit(Limiters, State); -ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) -> - case esockd_rate_limit:check(Num, Rl) of +ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) -> + case esockd_rate_limit:check(Cnt, Rl) of {0, Rl1} -> ensure_rate_limit(Limiters, setelement(Pos, State, Rl1)); {Pause, Rl1} -> @@ -377,36 +392,52 @@ ensure_rate_limit([{Rl, Pos, Num}|Limiters], State) -> setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1) end. +%%------------------------------------------------------------------------------ +%% Activate socket + run_socket(State = #state{conn_state = blocked}) -> State; -run_socket(State = #state{await_recv = true}) -> - State; -run_socket(State = #state{transport = Transport, socket = Socket}) -> - Transport:async_recv(Socket, 0, infinity), - State#state{await_recv = true}. + +run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) -> + TrueOrN = case Transport:is_ssl(Socket) of + true -> true; %% Cannot set '{active, N}' for SSL:( + false -> N + end, + ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])), + State. + +ensure_ok_or_exit(ok) -> ok; +ensure_ok_or_exit({error, Reason}) -> + self() ! {shutdown, Reason}. %%------------------------------------------------------------------------------ %% Ensure stats timer -%%------------------------------------------------------------------------------ ensure_stats_timer(State = #state{enable_stats = true, - stats_timer = undefined, + stats_timer = undefined, idle_timeout = IdleTimeout}) -> State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; ensure_stats_timer(State) -> State. +%%------------------------------------------------------------------------------ +%% Maybe GC + +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) -> + Oct = iolist_size(Payload), + maybe_gc({1, Oct}, State); +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}; +maybe_gc(_, State) -> + State. + +%%------------------------------------------------------------------------------ +%% Shutdown or stop + shutdown(Reason, State) -> stop({shutdown, Reason}, State). stop(Reason, State) -> {stop, Reason, State}. - -%% For incoming messages, bump gc-stats with packet count and totoal volume -%% For outgoing messages, only 'publish' type is taken into account. -maybe_gc(#state{incoming = #{bytes := Oct, packets := Cnt}}, incoming) -> - ok = emqx_gc:inc(Cnt, Oct); -maybe_gc(#state{}, {publish, _PacketId, #message{payload = Payload}}) -> - Oct = iolist_size(Payload), - ok = emqx_gc:inc(1, Oct); -maybe_gc(_, _) -> - ok. diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl index 17166a014..1d2fb13a3 100644 --- a/src/emqx_ctl.erl +++ b/src/emqx_ctl.erl @@ -54,15 +54,6 @@ run_command([Cmd | Args]) -> run_command(list_to_atom(Cmd), Args). -spec(run_command(cmd(), [string()]) -> ok | {error, term()}). -% run_command(set, []) -> -% emqx_mgmt_cli_cfg:set_usage(), ok; - -% run_command(set, Args) -> -% emqx_mgmt_cli_cfg:run(["config" | Args]), ok; - -% run_command(show, Args) -> -% emqx_mgmt_cli_cfg:run(["config" | Args]), ok; - run_command(help, []) -> usage(); run_command(Cmd, Args) when is_atom(Cmd) -> @@ -96,7 +87,7 @@ usage() -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [ordered_set, protected]), + ok = emqx_tables:new(?TAB, [protected, ordered_set]), {ok, #state{seq = 0}}. handle_call(Req, _From, State) -> @@ -160,4 +151,3 @@ register_command_test_() -> }. -endif. - diff --git a/src/emqx_gc.erl b/src/emqx_gc.erl index 7e98eb37a..d608954a0 100644 --- a/src/emqx_gc.erl +++ b/src/emqx_gc.erl @@ -21,74 +21,83 @@ -module(emqx_gc). --export([init/1, inc/2, reset/0]). +-export([init/1, run/3, info/1, reset/1]). --type st() :: #{ cnt => {integer(), integer()} - , oct => {integer(), integer()} - }. +-type(opts() :: #{count => integer(), + bytes => integer()}). + +-type(st() :: #{cnt => {integer(), integer()}, + oct => {integer(), integer()}}). + +-type(gc_state() :: {?MODULE, st()}). -define(disabled, disabled). -define(ENABLED(X), (is_integer(X) andalso X > 0)). -%% @doc Initialize force GC parameters. --spec init(false | map()) -> ok. +%% @doc Initialize force GC state. +-spec(init(opts() | false) -> gc_state() | undefined). init(#{count := Count, bytes := Bytes}) -> Cnt = [{cnt, {Count, Count}} || ?ENABLED(Count)], Oct = [{oct, {Bytes, Bytes}} || ?ENABLED(Bytes)], - erlang:put(?MODULE, maps:from_list(Cnt ++ Oct)), - ok; -init(_) -> erlang:put(?MODULE, #{}), ok. + {?MODULE, maps:from_list(Cnt ++ Oct)}; +init(false) -> undefined. -%% @doc Increase count and bytes stats in one call, -%% ensure gc is triggered at most once, even if both thresholds are hit. --spec inc(pos_integer(), pos_integer()) -> ok. -inc(Cnt, Oct) -> - mutate_pd_with(fun(St) -> inc(St, Cnt, Oct) end). +%% @doc Try to run GC based on reduntions of count or bytes. +-spec(run(pos_integer(), pos_integer(), gc_state()) -> {boolean(), gc_state()}). +run(Cnt, Oct, {?MODULE, St}) -> + {Res, St1} = run([{cnt, Cnt}, {oct, Oct}], St), + {Res, {?MODULE, St1}}; +run(_Cnt, _Oct, undefined) -> + {false, undefined}. -%% @doc Reset counters to zero. --spec reset() -> ok. -reset() -> - mutate_pd_with(fun(St) -> reset(St) end). - -%% ======== Internals ======== - -%% mutate gc stats numbers in process dict with the given function -mutate_pd_with(F) -> - St = F(erlang:get(?MODULE)), - erlang:put(?MODULE, St), - ok. - -%% Increase count and bytes stats in one call, -%% ensure gc is triggered at most once, even if both thresholds are hit. --spec inc(st(), pos_integer(), pos_integer()) -> st(). -inc(St0, Cnt, Oct) -> - case do_inc(St0, cnt, Cnt) of - {true, St} -> - St; +run([], St) -> + {false, St}; +run([{K, N}|T], St) -> + case dec(K, N, St) of + {true, St1} -> + {true, do_gc(St1)}; {false, St1} -> - {_, St} = do_inc(St1, oct, Oct), - St + run(T, St1) end. -%% Reset counters to zero. -reset(St) -> reset(cnt, reset(oct, St)). +%% @doc Info of GC state. +-spec(info(gc_state()) -> map() | undefined). +info({?MODULE, St}) -> + St; +info(undefined) -> + undefined. --spec do_inc(st(), cnt | oct, pos_integer()) -> {boolean(), st()}. -do_inc(St, Key, Num) -> +%% @doc Reset counters to zero. +-spec(reset(gc_state()) -> gc_state()). +reset({?MODULE, St}) -> + {?MODULE, do_reset(St)}; +reset(undefined) -> + undefined. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +-spec(dec(cnt | oct, pos_integer(), st()) -> {boolean(), st()}). +dec(Key, Num, St) -> case maps:get(Key, St, ?disabled) of ?disabled -> {false, St}; {Init, Remain} when Remain > Num -> {false, maps:put(Key, {Init, Remain - Num}, St)}; _ -> - {true, do_gc(St)} + {true, St} end. do_gc(St) -> - erlang:garbage_collect(), - reset(St). + true = erlang:garbage_collect(), + do_reset(St). -reset(Key, St) -> +do_reset(St) -> + do_reset(cnt, do_reset(oct, St)). + +%% Reset counters to zero. +do_reset(Key, St) -> case maps:get(Key, St, ?disabled) of ?disabled -> St; {Init, _} -> maps:put(Key, {Init, Init}, St) diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl index b10445742..55b2476ee 100644 --- a/src/emqx_hooks.erl +++ b/src/emqx_hooks.erl @@ -139,7 +139,7 @@ lookup(HookPoint) -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = Action}}, _From, State) -> diff --git a/src/emqx_local_bridge.erl b/src/emqx_local_bridge.erl index 7c4e7cea1..df2dda686 100644 --- a/src/emqx_local_bridge.erl +++ b/src/emqx_local_bridge.erl @@ -61,7 +61,7 @@ init([Pool, Id, Node, Topic, Options]) -> true -> true = erlang:monitor_node(Node, true), Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), - emqx_broker:subscribe(Topic, self(), #{share => Group, qos => ?QOS_0}), + emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}), State = parse_opts(Options, #state{node = Node, subtopic = Topic}), MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len, store_qos0 => true}), diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index caf862146..b4b3a1307 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -285,7 +285,7 @@ qos_sent(?QOS_2) -> init([]) -> % Create metrics table - _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]), + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), lists:foreach(fun new/1, ?BYTES_METRICS ++ ?PACKET_METRICS ++ ?MESSAGE_METRICS), {ok, #{}, hibernate}. diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index cf4a555ca..38bc1c2b0 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -19,6 +19,8 @@ -export([init_proc_mng_policy/1, conn_proc_mng_policy/1]). +-export([drain_down/1]). + %% @doc Merge options -spec(merge_opts(list(), list()) -> list()). merge_opts(Defaults, Options) -> @@ -108,3 +110,19 @@ is_enabled(Max) -> is_integer(Max) andalso Max > ?DISABLED. proc_info(Key) -> {Key, Value} = erlang:process_info(self(), Key), Value. + +-spec(drain_down(pos_integer()) -> list(pid())). +drain_down(Cnt) when Cnt > 0 -> + drain_down(Cnt, []). + +drain_down(0, Acc) -> + lists:reverse(Acc); + +drain_down(Cnt, Acc) -> + receive + {'DOWN', _MRef, process, Pid, _Reason} -> + drain_down(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl index 56390d412..48b0fa439 100644 --- a/src/emqx_mqueue.erl +++ b/src/emqx_mqueue.erl @@ -5,7 +5,8 @@ %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 -%%%% Unless required by applicable law or agreed to in writing, software +%% +%% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and diff --git a/src/emqx_pd.erl b/src/emqx_pd.erl new file mode 100644 index 000000000..ce1e7723c --- /dev/null +++ b/src/emqx_pd.erl @@ -0,0 +1,33 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc The utility functions for erlang process dictionary. +-module(emqx_pd). + +-export([update_counter/2, get_counter/1, reset_counter/1]). + +-type(key() :: term()). + +-spec(update_counter(key(), number()) -> undefined | number()). +update_counter(Key, Inc) -> + put(Key, get_counter(Key) + Inc). + +-spec(get_counter(key()) -> number()). +get_counter(Key) -> + case get(Key) of undefined -> 0; Cnt -> Cnt end. + +-spec(reset_counter(key()) -> number()). +reset_counter(Key) -> + case put(Key, 0) of undefined -> 0; Cnt -> Cnt end. + diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 44585a69e..b0f6456dc 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -1,18 +1,16 @@ -%%%=================================================================== -%%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved. -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%%=================================================================== +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. -module(emqx_plugins). diff --git a/src/emqx_pmon.erl b/src/emqx_pmon.erl index 9b874041e..a00212ed5 100644 --- a/src/emqx_pmon.erl +++ b/src/emqx_pmon.erl @@ -14,24 +14,27 @@ -module(emqx_pmon). +-compile({no_auto_import, [monitor/3]}). + -export([new/0]). -export([monitor/2, monitor/3]). -export([demonitor/2]). -export([find/2]). --export([erase/2]). - --compile({no_auto_import,[monitor/3]}). +-export([erase/2, erase_all/2]). +-export([count/1]). -type(pmon() :: {?MODULE, map()}). -export_type([pmon/0]). -spec(new() -> pmon()). -new() -> {?MODULE, maps:new()}. +new() -> + {?MODULE, maps:new()}. -spec(monitor(pid(), pmon()) -> pmon()). monitor(Pid, PM) -> - monitor(Pid, undefined, PM). + ?MODULE:monitor(Pid, undefined, PM). +-spec(monitor(pid(), term(), pmon()) -> pmon()). monitor(Pid, Val, {?MODULE, PM}) -> {?MODULE, case maps:is_key(Pid, PM) of true -> PM; @@ -43,21 +46,36 @@ monitor(Pid, Val, {?MODULE, PM}) -> demonitor(Pid, {?MODULE, PM}) -> {?MODULE, case maps:find(Pid, PM) of {ok, {Ref, _Val}} -> - %% Don't flush - _ = erlang:demonitor(Ref), + %% flush + _ = erlang:demonitor(Ref, [flush]), maps:remove(Pid, PM); error -> PM end}. --spec(find(pid(), pmon()) -> undefined | term()). +-spec(find(pid(), pmon()) -> error | {ok, term()}). find(Pid, {?MODULE, PM}) -> case maps:find(Pid, PM) of {ok, {_Ref, Val}} -> - Val; - error -> undefined + {ok, Val}; + error -> error end. -spec(erase(pid(), pmon()) -> pmon()). erase(Pid, {?MODULE, PM}) -> {?MODULE, maps:remove(Pid, PM)}. +-spec(erase_all([pid()], pmon()) -> {[{pid(), term()}], pmon()}). +erase_all(Pids, PMon0) -> + lists:foldl( + fun(Pid, {Acc, PMon}) -> + case find(Pid, PMon) of + {ok, Val} -> + {[{Pid, Val}|Acc], erase(Pid, PMon)}; + error -> {Acc, PMon} + end + end, {[], PMon0}, Pids). + +-spec(count(pmon()) -> non_neg_integer()). +count({?MODULE, PM}) -> + maps:size(PM). + diff --git a/src/emqx_pool_sup.erl b/src/emqx_pool_sup.erl index b371549c0..eb81233f9 100644 --- a/src/emqx_pool_sup.erl +++ b/src/emqx_pool_sup.erl @@ -37,9 +37,8 @@ spec(ChildId, Args) -> start_link(Pool, Type, MFA) -> start_link(Pool, Type, emqx_vm:schedulers(), MFA). --spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, term()}). -start_link(Pool, Type, Size, MFA) when is_atom(Pool) -> - supervisor:start_link({local, Pool}, ?MODULE, [Pool, Type, Size, MFA]); +-spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) + -> {ok, pid()} | {error, term()}). start_link(Pool, Type, Size, MFA) -> supervisor:start_link(?MODULE, [Pool, Type, Size, MFA]). diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 1f9d19094..5495e4ff7 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -65,7 +65,8 @@ send_stats, connected, connected_at, - ignore_loop + ignore_loop, + topic_alias_maximum }). -type(state() :: #pstate{}). @@ -85,29 +86,30 @@ -spec(init(map(), list()) -> state()). init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) -> Zone = proplists:get_value(zone, Options), - #pstate{zone = Zone, - sendfun = SendFun, - peername = Peername, - peercert = Peercert, - proto_ver = ?MQTT_PROTO_V4, - proto_name = <<"MQTT">>, - client_id = <<>>, - is_assigned = false, - conn_pid = self(), - username = init_username(Peercert, Options), - is_super = false, - clean_start = false, - topic_aliases = #{}, - packet_size = emqx_zone:get_env(Zone, max_packet_size), - mountpoint = emqx_zone:get_env(Zone, mountpoint), - is_bridge = false, - enable_ban = emqx_zone:get_env(Zone, enable_ban, false), - enable_acl = emqx_zone:get_env(Zone, enable_acl), - acl_deny_action = emqx_zone:get_env(Zone, acl_deny_action, ignore), - recv_stats = #{msg => 0, pkt => 0}, - send_stats = #{msg => 0, pkt => 0}, - connected = false, - ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false)}. + #pstate{zone = Zone, + sendfun = SendFun, + peername = Peername, + peercert = Peercert, + proto_ver = ?MQTT_PROTO_V4, + proto_name = <<"MQTT">>, + client_id = <<>>, + is_assigned = false, + conn_pid = self(), + username = init_username(Peercert, Options), + is_super = false, + clean_start = false, + topic_aliases = #{}, + packet_size = emqx_zone:get_env(Zone, max_packet_size), + mountpoint = emqx_zone:get_env(Zone, mountpoint), + is_bridge = false, + enable_ban = emqx_zone:get_env(Zone, enable_ban, false), + enable_acl = emqx_zone:get_env(Zone, enable_acl), + acl_deny_action = emqx_zone:get_env(Zone, acl_deny_action, ignore), + recv_stats = #{msg => 0, pkt => 0}, + send_stats = #{msg => 0, pkt => 0}, + connected = false, + ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false), + topic_alias_maximum = #{to_client => 0, from_client => 0}}. init_username(Peercert, Options) -> case proplists:get_value(peer_cert_as_username, Options) of @@ -214,12 +216,16 @@ received(?PACKET(?CONNECT), PState = #pstate{connected = true}) -> {error, proto_unexpected_connect, PState}; received(Packet = ?PACKET(Type), PState) -> - PState1 = set_protover(Packet, PState), + PState1 = set_protover(Packet, PState), trace(recv, Packet), try emqx_packet:validate(Packet) of true -> - {Packet1, PState2} = preprocess_properties(Packet, PState1), - process_packet(Packet1, inc_stats(recv, Type, PState2)) + case preprocess_properties(Packet, PState1) of + {error, ReasonCode} -> + {error, ReasonCode, PState1}; + {Packet1, PState2} -> + process_packet(Packet1, inc_stats(recv, Type, PState2)) + end catch error : protocol_error -> deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState1), @@ -244,6 +250,13 @@ received(Packet = ?PACKET(Type), PState) -> %%------------------------------------------------------------------------------ %% Preprocess MQTT Properties %%------------------------------------------------------------------------------ +preprocess_properties(Packet = #mqtt_packet{ + variable = #mqtt_packet_connect{ + properties = #{'Topic-Alias-Maximum' := ToClient} + } + }, + PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) -> + {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}}; %% Subscription Identifier preprocess_properties(Packet = #mqtt_packet{ @@ -257,22 +270,46 @@ preprocess_properties(Packet = #mqtt_packet{ {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState}; %% Topic Alias Mapping +preprocess_properties(#mqtt_packet{ + variable = #mqtt_packet_publish{ + properties = #{'Topic-Alias' := 0}} + }, + PState) -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID}; + preprocess_properties(Packet = #mqtt_packet{ variable = Publish = #mqtt_packet_publish{ topic_name = <<>>, properties = #{'Topic-Alias' := AliasId}} }, - PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) -> - {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ - topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ + topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; preprocess_properties(Packet = #mqtt_packet{ - variable = #mqtt_packet_publish{ - topic_name = Topic, - properties = #{'Topic-Alias' := AliasId}} - }, - PState = #pstate{proto_ver = ?MQTT_PROTO_V5, topic_aliases = Aliases}) -> - {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + variable = #mqtt_packet_publish{ + topic_name = Topic, + properties = #{'Topic-Alias' := AliasId}} + }, + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; preprocess_properties(Packet, PState) -> {Packet, PState}. @@ -280,7 +317,6 @@ preprocess_properties(Packet, PState) -> %%------------------------------------------------------------------------------ %% Process MQTT Packet %%------------------------------------------------------------------------------ - process_packet(?CONNECT_PACKET( #mqtt_packet_connect{proto_name = ProtoName, proto_ver = ProtoVer, @@ -308,6 +344,7 @@ process_packet(?CONNECT_PACKET( conn_props = ConnProps, is_bridge = IsBridge, connected_at = os:timestamp()}), + connack( case check_connect(ConnPkt, PState1) of {ok, PState2} -> @@ -321,7 +358,8 @@ process_packet(?CONNECT_PACKET( case try_open_session(PState3) of {ok, SPid, SP} -> PState4 = PState3#pstate{session = SPid, connected = true}, - ok = emqx_cm:register_connection(client_id(PState4), attrs(PState4)), + ok = emqx_cm:register_connection(client_id(PState4)), + true = emqx_cm:set_conn_attrs(client_id(PState4), attrs(PState4)), %% Start keepalive start_keepalive(Keepalive, PState4), %% Success @@ -507,18 +545,18 @@ do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId), puback(?QOS_0, _PacketId, _Result, PState) -> {ok, PState}; +puback(?QOS_1, PacketId, [], PState) -> + deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); +puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({puback, PacketId, ?RC_SUCCESS}, PState); puback(?QOS_1, PacketId, {error, ReasonCode}, PState) -> deliver({puback, PacketId, ReasonCode}, PState); -puback(?QOS_1, PacketId, {ok, []}, PState) -> - deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_1, PacketId, {ok, _}, PState) -> - deliver({puback, PacketId, ?RC_SUCCESS}, PState); -puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> - deliver({pubrec, PacketId, ReasonCode}, PState); -puback(?QOS_2, PacketId, {ok, []}, PState) -> +puback(?QOS_2, PacketId, [], PState) -> deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_2, PacketId, {ok, _}, PState) -> - deliver({pubrec, PacketId, ?RC_SUCCESS}, PState). +puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({pubrec, PacketId, ?RC_SUCCESS}, PState); +puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> + deliver({pubrec, PacketId, ReasonCode}, PState). %%------------------------------------------------------------------------------ %% Deliver Packet -> Client @@ -532,7 +570,8 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone, proto_ver = ?MQTT_PROTO_V5, client_id = ClientId, conn_props = ConnProps, - is_assigned = IsAssigned}) -> + is_assigned = IsAssigned, + topic_alias_maximum = TopicAliasMaximum}) -> ResponseInformation = case maps:find('Request-Response-Information', ConnProps) of {ok, 1} -> iolist_to_binary(emqx_config:get_env(response_topic_prefix)); @@ -570,17 +609,20 @@ deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone, undefined -> Props2; Keepalive -> Props2#{'Server-Keep-Alive' => Keepalive} end, - send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState); + + PState1 = PState#pstate{topic_alias_maximum = TopicAliasMaximum#{from_client => MaxAlias}}, + + send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState1); deliver({connack, ReasonCode, SP}, PState) -> send(?CONNACK_PACKET(ReasonCode, SP), PState); -deliver({publish, PacketId, Msg}, PState = #pstate{mountpoint = MountPoint}) -> +deliver({publish, PacketId, Msg = #message{headers = Headers}}, PState = #pstate{mountpoint = MountPoint}) -> _ = emqx_hooks:run('message.delivered', [credentials(PState)], Msg), Msg1 = emqx_message:update_expiry(Msg), Msg2 = emqx_mountpoint:unmount(MountPoint, Msg1), - send(emqx_packet:from_message(PacketId, Msg2), PState); - + send(emqx_packet:from_message(PacketId, Msg2#message{headers = maps:remove('Topic-Alias', Headers)}), PState); + deliver({puback, PacketId, ReasonCode}, PState) -> send(?PUBACK_PACKET(PacketId, ReasonCode), PState); @@ -629,14 +671,12 @@ maybe_use_username_as_clientid(ClientId, undefined, _PState) -> ClientId; maybe_use_username_as_clientid(ClientId, Username, #pstate{zone = Zone}) -> case emqx_zone:get_env(Zone, use_username_as_clientid, false) of - true -> - Username; - false -> - ClientId + true -> Username; + false -> ClientId end. %%------------------------------------------------------------------------------ -%% Assign a clientid +%% Assign a clientId maybe_assign_client_id(PState = #pstate{client_id = <<>>, ack_props = AckProps}) -> ClientId = emqx_guid:to_base62(emqx_guid:gen()), @@ -660,41 +700,37 @@ try_open_session(PState = #pstate{zone = Zone, clean_start => CleanStart, will_msg => WillMsg }, - SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}, {topic_alias_maximum, PState}]), + SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}]), case emqx_sm:open_session(SessAttrs1) of {ok, SPid} -> {ok, SPid, false}; Other -> Other end. -set_session_attrs({max_inflight, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) -> - maps:put(max_inflight, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Receive-Maximum', ConnProps, 65535); - true -> - emqx_zone:get_env(Zone, max_inflight, 65535) - end, SessAttrs); -set_session_attrs({expiry_interval, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps, clean_start = CleanStart}}, SessAttrs) -> - maps:put(expiry_interval, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Session-Expiry-Interval', ConnProps, 0); - true -> - case CleanStart of - true -> 0; - false -> - emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff) - end - end, SessAttrs); -set_session_attrs({topic_alias_maximum, #pstate{zone = Zone, proto_ver = ProtoVer, conn_props = ConnProps}}, SessAttrs) -> - maps:put(topic_alias_maximum, if - ProtoVer =:= ?MQTT_PROTO_V5 -> - get_property('Topic-Alias-Maximum', ConnProps, 0); - true -> - emqx_zone:get_env(Zone, max_topic_alias, 0) - end, SessAttrs); -set_session_attrs({_, #pstate{}}, SessAttrs) -> - SessAttrs. +set_session_attrs({max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(max_inflight, get_property('Receive-Maximum', ConnProps, 65535), SessAttrs); + +set_session_attrs({max_inflight, #pstate{zone = Zone}}, SessAttrs) -> + maps:put(max_inflight, emqx_zone:get_env(Zone, max_inflight, 65535), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(expiry_interval, get_property('Session-Expiry-Interval', ConnProps, 0), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{zone = Zone, clean_start = CleanStart}}, SessAttrs) -> + maps:put(expiry_interval, case CleanStart of + true -> 0; + false -> emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff) + end, SessAttrs); + +set_session_attrs({topic_alias_maximum, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(topic_alias_maximum, get_property('Topic-Alias-Maximum', ConnProps, 0), SessAttrs); + +set_session_attrs({topic_alias_maximum, #pstate{zone = Zone}}, SessAttrs) -> + maps:put(topic_alias_maximum, emqx_zone:get_env(Zone, max_topic_alias, 0), SessAttrs); + +set_session_attrs(_, SessAttrs) -> + SessAttrs. authenticate(Credentials, Password) -> case emqx_access_control:authenticate(Credentials, Password) of @@ -803,12 +839,6 @@ check_publish(Packet, PState) -> run_check_steps([fun check_pub_caps/2, fun check_pub_acl/2], Packet, PState). -check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ - properties = #{'Topic-Alias' := TopicAlias} - }}, - #pstate{zone = Zone}) -> - emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}); check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{ properties = _Properties}}, #pstate{zone = Zone}) -> @@ -881,14 +911,13 @@ shutdown(_Reason, #pstate{client_id = undefined}) -> ok; shutdown(_Reason, #pstate{connected = false}) -> ok; -shutdown(Reason, #pstate{client_id = ClientId}) when Reason =:= conflict; - Reason =:= discard -> - emqx_cm:unregister_connection(ClientId); -shutdown(Reason, PState = #pstate{connected = true, - client_id = ClientId}) -> +shutdown(conflict, _PState) -> + ok; +shutdown(discard, _PState) -> + ok; +shutdown(Reason, PState) -> ?LOG(info, "Shutdown for ~p", [Reason]), - emqx_hooks:run('client.disconnected', [credentials(PState), Reason]), - emqx_cm:unregister_connection(ClientId). + emqx_hooks:run('client.disconnected', [credentials(PState), Reason]). start_keepalive(0, _PState) -> ignore; diff --git a/src/emqx_rate_limiter.erl b/src/emqx_rate_limiter.erl index 76eaf0c61..c8145dbbc 100644 --- a/src/emqx_rate_limiter.erl +++ b/src/emqx_rate_limiter.erl @@ -1,18 +1,16 @@ -%%%------------------------------------------------------------------- -%%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%%------------------------------------------------------------------- +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. -module(emqx_rate_limiter). diff --git a/src/emqx_router.erl b/src/emqx_router.erl index e513d041d..0463526d6 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -28,23 +28,22 @@ -export([start_link/2]). %% Route APIs --export([add_route/1, add_route/2, add_route/3]). --export([get_routes/1]). --export([del_route/1, del_route/2, del_route/3]). --export([has_routes/1, match_routes/1, print_routes/1]). +-export([add_route/1, add_route/2]). +-export([do_add_route/1, do_add_route/2]). +-export([match_routes/1, lookup_routes/1, has_routes/1]). +-export([delete_route/1, delete_route/2]). +-export([do_delete_route/1, do_delete_route/2]). +-export([print_routes/1]). -export([topics/0]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --type(destination() :: node() | {binary(), node()}). - --record(batch, {enabled, timer, pending}). --record(state, {pool, id, batch :: #batch{}}). +-type(group() :: binary()). +-type(destination() :: node() | {group(), node()}). -define(ROUTE, emqx_route). --define(BATCH(Enabled), #batch{enabled = Enabled}). --define(BATCH(Enabled, Pending), #batch{enabled = Enabled, pending = Pending}). %%------------------------------------------------------------------------------ %% Mnesia bootstrap @@ -62,10 +61,10 @@ mnesia(copy) -> ok = ekka_mnesia:copy_table(?ROUTE). %%------------------------------------------------------------------------------ -%% Strat a router +%% Start a router %%------------------------------------------------------------------------------ --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). start_link(Pool, Id) -> gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). @@ -74,51 +73,69 @@ start_link(Pool, Id) -> %% Route APIs %%------------------------------------------------------------------------------ --spec(add_route(emqx_topic:topic() | emqx_types:route()) -> ok). +-spec(add_route(emqx_topic:topic()) -> ok | {error, term()}). add_route(Topic) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = node()}); -add_route(Route = #route{topic = Topic}) -> - cast(pick(Topic), {add_route, Route}). + add_route(Topic, node()). --spec(add_route(emqx_topic:topic(), destination()) -> ok). +-spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). add_route(Topic, Dest) when is_binary(Topic) -> - add_route(#route{topic = Topic, dest = Dest}). + call(pick(Topic), {add_route, Topic, Dest}). --spec(add_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok). -add_route(From, Topic, Dest) when is_binary(Topic) -> - cast(pick(Topic), {add_route, From, #route{topic = Topic, dest = Dest}}). +-spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}). +do_add_route(Topic) when is_binary(Topic) -> + do_add_route(Topic, node()). --spec(get_routes(emqx_topic:topic()) -> [emqx_types:route()]). -get_routes(Topic) -> +-spec(do_add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_add_route(Topic, Dest) when is_binary(Topic) -> + Route = #route{topic = Topic, dest = Dest}, + case lists:member(Route, lookup_routes(Topic)) of + true -> ok; + false -> + ok = emqx_router_helper:monitor(Dest), + case emqx_topic:wildcard(Topic) of + true -> trans(fun insert_trie_route/1, [Route]); + false -> insert_direct_route(Route) + end + end. + +%% @doc Match routes +-spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). +match_routes(Topic) when is_binary(Topic) -> + %% Optimize: routing table will be replicated to all router nodes. + Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), + lists:append([lookup_routes(To) || To <- [Topic | Matched]]). + +-spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). +lookup_routes(Topic) -> ets:lookup(?ROUTE, Topic). --spec(del_route(emqx_topic:topic() | emqx_types:route()) -> ok). -del_route(Topic) when is_binary(Topic) -> - del_route(#route{topic = Topic, dest = node()}); -del_route(Route = #route{topic = Topic}) -> - cast(pick(Topic), {del_route, Route}). - --spec(del_route(emqx_topic:topic(), destination()) -> ok). -del_route(Topic, Dest) when is_binary(Topic) -> - del_route(#route{topic = Topic, dest = Dest}). - --spec(del_route({pid(), reference()}, emqx_topic:topic(), destination()) -> ok). -del_route(From, Topic, Dest) when is_binary(Topic) -> - cast(pick(Topic), {del_route, From, #route{topic = Topic, dest = Dest}}). - -spec(has_routes(emqx_topic:topic()) -> boolean()). has_routes(Topic) when is_binary(Topic) -> ets:member(?ROUTE, Topic). --spec(topics() -> list(emqx_topic:topic())). -topics() -> mnesia:dirty_all_keys(?ROUTE). +-spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}). +delete_route(Topic) when is_binary(Topic) -> + delete_route(Topic, node()). -%% @doc Match routes -%% Optimize: routing table will be replicated to all router nodes. --spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). -match_routes(Topic) when is_binary(Topic) -> - Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), - lists:append([get_routes(To) || To <- [Topic | Matched]]). +-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +delete_route(Topic, Dest) when is_binary(Topic) -> + call(pick(Topic), {delete_route, Topic, Dest}). + +-spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}). +do_delete_route(Topic) when is_binary(Topic) -> + do_delete_route(Topic, node()). + +-spec(do_delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_delete_route(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> trans(fun delete_trie_route/1, [Route]); + false -> delete_direct_route(Route) + end. + +-spec(topics() -> list(emqx_topic:topic())). +topics() -> + mnesia:dirty_all_keys(?ROUTE). %% @doc Print routes to a topic -spec(print_routes(emqx_topic:topic()) -> ok). @@ -127,82 +144,41 @@ print_routes(Topic) -> io:format("~s -> ~s~n", [To, Dest]) end, match_routes(Topic)). -cast(Router, Msg) -> - gen_server:cast(Router, Msg). +call(Router, Msg) -> + gen_server:call(Router, Msg, infinity). pick(Topic) -> - gproc_pool:pick_worker(router, Topic). + gproc_pool:pick_worker(router_pool, Topic). %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ init([Pool, Id]) -> - rand:seed(exsplus, erlang:timestamp()), - gproc_pool:connect_worker(Pool, {Pool, Id}), - Batch = #batch{enabled = emqx_config:get_env(route_batch_clean, false), - pending = sets:new()}, - {ok, ensure_batch_timer(#state{pool = Pool, id = Id, batch = Batch})}. + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +handle_call({add_route, Topic, Dest}, _From, State) -> + Ok = do_add_route(Topic, Dest), + {reply, Ok, State}; + +handle_call({delete_route, Topic, Dest}, _From, State) -> + Ok = do_delete_route(Topic, Dest), + {reply, Ok, State}; handle_call(Req, _From, State) -> emqx_logger:error("[Router] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({add_route, From, Route}, State) -> - {noreply, NewState} = handle_cast({add_route, Route}, State), - _ = gen_server:reply(From, ok), - {noreply, NewState}; - -handle_cast({add_route, Route = #route{topic = Topic, dest = Dest}}, State) -> - case lists:member(Route, get_routes(Topic)) of - true -> ok; - false -> - ok = emqx_router_helper:monitor(Dest), - case emqx_topic:wildcard(Topic) of - true -> log(trans(fun add_trie_route/1, [Route])); - false -> add_direct_route(Route) - end - end, - {noreply, State}; - -handle_cast({del_route, From, Route}, State) -> - {noreply, NewState} = handle_cast({del_route, Route}, State), - _ = gen_server:reply(From, ok), - {noreply, NewState}; - -handle_cast({del_route, Route = #route{topic = Topic, dest = Dest}}, State) when is_tuple(Dest) -> - {noreply, case emqx_topic:wildcard(Topic) of - true -> log(trans(fun del_trie_route/1, [Route])), - State; - false -> del_direct_route(Route, State) - end}; - -handle_cast({del_route, Route = #route{topic = Topic}}, State) -> - %% Confirm if there are still subscribers... - {noreply, case ets:member(emqx_subscriber, Topic) of - true -> State; - false -> - case emqx_topic:wildcard(Topic) of - true -> log(trans(fun del_trie_route/1, [Route])), - State; - false -> del_direct_route(Route, State) - end - end}; - handle_cast(Msg, State) -> emqx_logger:error("[Router] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({timeout, _TRef, batch_delete}, State = #state{batch = Batch}) -> - _ = del_direct_routes(sets:to_list(Batch#batch.pending)), - {noreply, ensure_batch_timer(State#state{batch = ?BATCH(true, sets:new())}), hibernate}; - handle_info(Info, State) -> emqx_logger:error("[Router] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{pool = Pool, id = Id, batch = Batch}) -> - _ = cacel_batch_timer(Batch), +terminate(_Reason, #{pool := Pool, id := Id}) -> gproc_pool:disconnect_worker(Pool, {Pool, Id}). code_change(_OldVsn, State, _Extra) -> @@ -212,50 +188,23 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ -ensure_batch_timer(State = #state{batch = #batch{enabled = false}}) -> - State; -ensure_batch_timer(State = #state{batch = Batch}) -> - TRef = erlang:start_timer(50 + rand:uniform(50), self(), batch_delete), - State#state{batch = Batch#batch{timer = TRef}}. - -cacel_batch_timer(#batch{enabled = false}) -> - ok; -cacel_batch_timer(#batch{enabled = true, timer = TRef}) -> - catch erlang:cancel_timer(TRef). - -add_direct_route(Route) -> +insert_direct_route(Route) -> mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]). -add_trie_route(Route = #route{topic = Topic}) -> +insert_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [] -> emqx_trie:insert(Topic); _ -> ok end, mnesia:write(?ROUTE, Route, sticky_write). -del_direct_route(Route, State = #state{batch = ?BATCH(false)}) -> - del_direct_route(Route), State; -del_direct_route(Route, State = #state{batch = Batch = ?BATCH(true, Pending)}) -> - State#state{batch = Batch#batch{pending = sets:add_element(Route, Pending)}}. - -del_direct_route(Route) -> +delete_direct_route(Route) -> mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]). -del_direct_routes([]) -> - ok; -del_direct_routes(Routes) -> - DelFun = fun(R = #route{topic = Topic}) -> - case ets:member(emqx_subscriber, Topic) of - true -> ok; - false -> mnesia:delete_object(?ROUTE, R, sticky_write) - end - end, - mnesia:async_dirty(fun lists:foreach/2, [DelFun, Routes]). - -del_trie_route(Route = #route{topic = Topic}) -> +delete_trie_route(Route = #route{topic = Topic}) -> case mnesia:wread({?ROUTE, Topic}) of [Route] -> %% Remove route and trie - mnesia:delete_object(?ROUTE, Route, sticky_write), + ok = mnesia:delete_object(?ROUTE, Route, sticky_write), emqx_trie:delete(Topic); [_|_] -> %% Remove route only mnesia:delete_object(?ROUTE, Route, sticky_write); @@ -266,11 +215,7 @@ del_trie_route(Route = #route{topic = Topic}) -> -spec(trans(function(), list(any())) -> ok | {error, term()}). trans(Fun, Args) -> case mnesia:transaction(Fun, Args) of - {atomic, _} -> ok; - {aborted, Error} -> {error, Error} + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} end. -log(ok) -> ok; -log({error, Reason}) -> - emqx_logger:error("[Router] mnesia aborted: ~p", [Reason]). - diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index c24b10715..c32800a24 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -31,15 +31,11 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -%% internal export +%% Internal export -export([stats_fun/0]). -record(routing_node, {name, const = unused}). --record(state, {nodes = []}). --compile({no_auto_import, [monitor/1]}). - --define(SERVER, ?MODULE). -define(ROUTE, emqx_route). -define(ROUTING_NODE, emqx_routing_node). -define(LOCK, {?MODULE, cleanup_routes}). @@ -64,9 +60,9 @@ mnesia(copy) -> %%------------------------------------------------------------------------------ %% @doc Starts the router helper --spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). %% @doc Monitor routing node -spec(monitor(node() | {binary(), node()}) -> ok). @@ -84,18 +80,18 @@ monitor(Node) when is_atom(Node) -> %%------------------------------------------------------------------------------ init([]) -> - _ = ekka:monitor(membership), - _ = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + ok = ekka:monitor(membership), + {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of true -> Acc; - false -> _ = erlang:monitor_node(Node, true), + false -> true = erlang:monitor_node(Node, true), [Node | Acc] end end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), - emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), - {ok, #state{nodes = Nodes}, hibernate}. + ok = emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), + {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> emqx_logger:error("[RouterHelper] unexpected call: ~p", [Req]), @@ -105,24 +101,29 @@ handle_cast(Msg, State) -> emqx_logger:error("[RouterHelper] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({mnesia_table_event, {write, #routing_node{name = Node}, _}}, State = #state{nodes = Nodes}) -> - emqx_logger:info("[RouterHelper] write routing node: ~s", [Node]), +handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, State = #{nodes := Nodes}) -> case ekka:is_member(Node) orelse lists:member(Node, Nodes) of - true -> {noreply, State}; - false -> _ = erlang:monitor_node(Node, true), - {noreply, State#state{nodes = [Node | Nodes]}} + true -> {noreply, State}; + false -> + true = erlang:monitor_node(Node, true), + {noreply, State#{nodes := [Node | Nodes]}} end; -handle_info({mnesia_table_event, _Event}, State) -> +handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> + %% ignore {noreply, State}; -handle_info({nodedown, Node}, State = #state{nodes = Nodes}) -> +handle_info({mnesia_table_event, Event}, State) -> + emqx_logger:error("[RouterHelper] unexpected mnesia_table_event: ~p", [Event]), + {noreply, State}; + +handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> global:trans({?LOCK, self()}, fun() -> mnesia:transaction(fun cleanup_routes/1, [Node]) end), - mnesia:dirty_delete(?ROUTING_NODE, Node), - {noreply, State#state{nodes = lists:delete(Node, Nodes)}, hibernate}; + ok = mnesia:dirty_delete(?ROUTING_NODE, Node), + {noreply, State#{nodes := lists:delete(Node, Nodes)}, hibernate}; handle_info({membership, {mnesia, down, Node}}, State) -> handle_info({nodedown, Node}, State); @@ -134,8 +135,8 @@ handle_info(Info, State) -> emqx_logger:error("[RouteHelper] unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #state{}) -> - ekka:unmonitor(membership), +terminate(_Reason, _State) -> + ok = ekka:unmonitor(membership), emqx_stats:cancel_update(route_stats), mnesia:unsubscribe({table, ?ROUTING_NODE, simple}). diff --git a/src/emqx_router_sup.erl b/src/emqx_router_sup.erl index 2bbaabc18..945d7910f 100644 --- a/src/emqx_router_sup.erl +++ b/src/emqx_router_sup.erl @@ -17,6 +17,7 @@ -behaviour(supervisor). -export([start_link/0]). + -export([init/1]). start_link() -> @@ -32,8 +33,7 @@ init([]) -> modules => [emqx_router_helper]}, %% Router pool - RouterPool = emqx_pool_sup:spec(emqx_router_pool, - [router, hash, emqx_vm:schedulers(), + RouterPool = emqx_pool_sup:spec([router_pool, hash, {emqx_router, start_link, []}]), {ok, {{one_for_all, 0, 1}, [Helper, RouterPool]}}. diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl new file mode 100644 index 000000000..33bb5edda --- /dev/null +++ b/src/emqx_sequence.erl @@ -0,0 +1,60 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence). + +-export([create/1, nextval/2, currval/2, reclaim/2, delete/1]). + +-type(key() :: term()). +-type(name() :: atom()). +-type(seqid() :: non_neg_integer()). + +-export_type([seqid/0]). + +%% @doc Create a sequence. +-spec(create(name()) -> ok). +create(Name) -> + emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + +%% @doc Next value of the sequence. +-spec(nextval(name(), key()) -> seqid()). +nextval(Name, Key) -> + ets:update_counter(Name, Key, {2, 1}, {Key, 0}). + +%% @doc Current value of the sequence. +-spec(currval(name(), key()) -> seqid()). +currval(Name, Key) -> + try ets:lookup_element(Name, Key, 2) + catch + error:badarg -> 0 + end. + +%% @doc Reclaim a sequence id. +-spec(reclaim(name(), key()) -> seqid()). +reclaim(Name, Key) -> + try ets:update_counter(Name, Key, {2, -1, 0, 0}) of + 0 -> ets:delete_object(Name, {Key, 0}), 0; + I -> I + catch + error:badarg -> 0 + end. + +%% @doc Delete the sequence. +-spec(delete(name()) -> boolean()). +delete(Name) -> + case ets:info(Name, name) of + Name -> ets:delete(Name); + undefined -> false + end. + diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 30b35bf11..3f9fd1610 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -144,11 +144,12 @@ %% Enqueue stats enqueue_stats = 0, + %% GC State + gc_state, + %% Created at created_at :: erlang:timestamp(), - topic_alias_maximum :: pos_integer(), - will_msg :: emqx:message(), will_delay_timer :: reference() | undefined @@ -160,8 +161,6 @@ -export_type([attr/0]). --define(TIMEOUT, 60000). - -define(LOG(Level, Format, Args, _State), emqx_logger:Level("[Session] " ++ Format, Args)). @@ -259,13 +258,15 @@ subscribe(SPid, PacketId, Properties, TopicFilters) -> %% @doc Called by connection processes when publishing messages -spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message()) - -> {ok, emqx_types:deliver_results()}). + -> emqx_types:deliver_results() | {error, term()}). publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) -> %% Publish QoS0 message directly emqx_broker:publish(Msg); + publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) -> %% Publish QoS1 message directly emqx_broker:publish(Msg); + publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> %% Register QoS2 message packet ID (and timestamp) to session, then publish case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of @@ -277,6 +278,7 @@ publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> puback(SPid, PacketId) -> gen_server:cast(SPid, {puback, PacketId, ?RC_SUCCESS}). +-spec(puback(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) -> ok). puback(SPid, PacketId, ReasonCode) -> gen_server:cast(SPid, {puback, PacketId, ReasonCode}). @@ -324,7 +326,7 @@ discard(SPid, ByPid) -> -spec(update_expiry_interval(spid(), timeout()) -> ok). update_expiry_interval(SPid, Interval) -> - gen_server:cast(SPid, {expiry_interval, Interval}). + gen_server:cast(SPid, {update_expiry_interval, Interval}). -spec(close(spid()) -> ok). close(SPid) -> @@ -334,47 +336,46 @@ close(SPid) -> %% gen_server callbacks %%------------------------------------------------------------------------------ -init([Parent, #{zone := Zone, - client_id := ClientId, - username := Username, - conn_pid := ConnPid, - clean_start := CleanStart, - expiry_interval := ExpiryInterval, - max_inflight := MaxInflight, - topic_alias_maximum := TopicAliasMaximum, - will_msg := WillMsg}]) -> - emqx_logger:set_metadata_client_id(ClientId), +init([Parent, #{zone := Zone, + client_id := ClientId, + username := Username, + conn_pid := ConnPid, + clean_start := CleanStart, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight, + will_msg := WillMsg}]) -> process_flag(trap_exit, true), true = link(ConnPid), - IdleTimout = get_env(Zone, idle_timeout, 30000), - State = #state{idle_timeout = IdleTimout, - clean_start = CleanStart, - binding = binding(ConnPid), - client_id = ClientId, - username = Username, - conn_pid = ConnPid, - subscriptions = #{}, - max_subscriptions = get_env(Zone, max_subscriptions, 0), - upgrade_qos = get_env(Zone, upgrade_qos, false), - inflight = emqx_inflight:new(MaxInflight), - mqueue = init_mqueue(Zone), - retry_interval = get_env(Zone, retry_interval, 0), - awaiting_rel = #{}, - await_rel_timeout = get_env(Zone, await_rel_timeout), - max_awaiting_rel = get_env(Zone, max_awaiting_rel), - expiry_interval = ExpiryInterval, - enable_stats = get_env(Zone, enable_stats, true), - deliver_stats = 0, - enqueue_stats = 0, - created_at = os:timestamp(), - topic_alias_maximum = TopicAliasMaximum, - will_msg = WillMsg - }, - emqx_sm:register_session(ClientId, attrs(State)), - emqx_sm:set_session_stats(ClientId, stats(State)), - emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), + emqx_logger:set_metadata_client_id(ClientId), GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), - ok = emqx_gc:init(GcPolicy), + IdleTimout = get_env(Zone, idle_timeout, 30000), + State = #state{idle_timeout = IdleTimout, + clean_start = CleanStart, + binding = binding(ConnPid), + client_id = ClientId, + username = Username, + conn_pid = ConnPid, + subscriptions = #{}, + max_subscriptions = get_env(Zone, max_subscriptions, 0), + upgrade_qos = get_env(Zone, upgrade_qos, false), + inflight = emqx_inflight:new(MaxInflight), + mqueue = init_mqueue(Zone), + retry_interval = get_env(Zone, retry_interval, 0), + awaiting_rel = #{}, + await_rel_timeout = get_env(Zone, await_rel_timeout), + max_awaiting_rel = get_env(Zone, max_awaiting_rel), + expiry_interval = ExpiryInterval, + enable_stats = get_env(Zone, enable_stats, true), + deliver_stats = 0, + enqueue_stats = 0, + gc_state = emqx_gc:init(GcPolicy), + created_at = os:timestamp(), + will_msg = WillMsg + }, + ok = emqx_sm:register_session(ClientId, self()), + true = emqx_sm:set_session_attrs(ClientId, attrs(State)), + true = emqx_sm:set_session_stats(ClientId, stats(State)), + emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), ok = emqx_misc:init_proc_mng_policy(Zone), ok = proc_lib:init_ack(Parent, {ok, self()}), gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State). @@ -400,53 +401,56 @@ handle_call(stats, _From, State) -> handle_call({discard, ByPid}, _From, State = #state{conn_pid = undefined}) -> ?LOG(warning, "Discarded by ~p", [ByPid], State), - {stop, {shutdown, discard}, ok, State}; + {stop, discarded, ok, State}; handle_call({discard, ByPid}, _From, State = #state{client_id = ClientId, conn_pid = ConnPid}) -> ?LOG(warning, "Conn ~p is discarded by ~p", [ConnPid, ByPid], State), ConnPid ! {shutdown, discard, {ClientId, ByPid}}, - {stop, {shutdown, discard}, ok, State}; + {stop, discarded, ok, State}; %% PUBLISH: This is only to register packetId to session state. %% The actual message dispatching should be done by the caller (e.g. connection) process. handle_call({register_publish_packet_id, PacketId, Ts}, _From, State = #state{awaiting_rel = AwaitingRel}) -> - reply(case is_awaiting_full(State) of - false -> - case maps:is_key(PacketId, AwaitingRel) of - true -> - {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; - false -> - State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, - {ok, ensure_await_rel_timer(State1)} - end; - true -> - emqx_metrics:trans(inc, 'messages/qos2/dropped'), - ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), - {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State} - end); + reply( + case is_awaiting_full(State) of + false -> + case maps:is_key(PacketId, AwaitingRel) of + true -> + {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; + false -> + State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, + {ok, ensure_stats_timer(ensure_await_rel_timer(State1))} + end; + true -> + ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), + emqx_metrics:trans(inc, 'messages/qos2/dropped'), + {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State} + end); %% PUBREC: handle_call({pubrec, PacketId, _ReasonCode}, _From, State = #state{inflight = Inflight}) -> - reply(case emqx_inflight:contain(PacketId, Inflight) of - true -> - {ok, acked(pubrec, PacketId, State)}; - false -> - emqx_metrics:trans(inc, 'packets/pubrec/missed'), - ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), - {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} - end); + reply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + {ok, ensure_stats_timer(acked(pubrec, PacketId, State))}; + false -> + ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrec/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); %% PUBREL: handle_call({pubrel, PacketId, _ReasonCode}, _From, State = #state{awaiting_rel = AwaitingRel}) -> - reply(case maps:take(PacketId, AwaitingRel) of - {_Ts, AwaitingRel1} -> - {ok, State#state{awaiting_rel = AwaitingRel1}}; - error -> - emqx_metrics:trans(inc, 'packets/pubrel/missed'), - ?LOG(warning, "Cannot find PUBREL: ~w", [PacketId], State), - {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} - end); + reply( + case maps:take(PacketId, AwaitingRel) of + {_Ts, AwaitingRel1} -> + {ok, ensure_stats_timer(State#state{awaiting_rel = AwaitingRel1})}; + error -> + ?LOG(warning, "The PUBREL PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrel/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); handle_call(close, _From, State) -> {stop, normal, ok, State}; @@ -465,7 +469,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), SubMap; {ok, _SubOpts} -> - emqx_broker:set_subopts(Topic, {self(), ClientId}, SubOpts), + emqx_broker:set_subopts(Topic, SubOpts), %% Why??? emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), maps:put(Topic, SubOpts, SubMap); @@ -476,7 +480,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, end} end, {[], Subscriptions}, TopicFilters), suback(FromPid, PacketId, ReasonCodes), - noreply(State#state{subscriptions = Subscriptions1}); + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); %% UNSUBSCRIBE: handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, @@ -485,7 +489,7 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, lists:foldr(fun({Topic, _SubOpts}, {Acc, SubMap}) -> case maps:find(Topic, SubMap) of {ok, SubOpts} -> - ok = emqx_broker:unsubscribe(Topic, ClientId), + ok = emqx_broker:unsubscribe(Topic), emqx_hooks:run('session.unsubscribed', [#{client_id => ClientId}, Topic, SubOpts]), {[?RC_SUCCESS|Acc], maps:remove(Topic, SubMap)}; error -> @@ -493,47 +497,50 @@ handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, end end, {[], Subscriptions}, TopicFilters), unsuback(From, PacketId, ReasonCodes), - noreply(State#state{subscriptions = Subscriptions1}); + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); %% PUBACK: handle_cast({puback, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> - case emqx_inflight:contain(PacketId, Inflight) of - true -> - noreply(dequeue(acked(puback, PacketId, State))); - false -> - ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), - emqx_metrics:trans(inc, 'packets/puback/missed'), - {noreply, State} - end; + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + ensure_stats_timer(dequeue(acked(puback, PacketId, State))); + false -> + ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/puback/missed'), + State + end); %% PUBCOMP: handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> - case emqx_inflight:contain(PacketId, Inflight) of - true -> - noreply(dequeue(acked(pubcomp, PacketId, State))); - false -> - ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), - emqx_metrics:trans(inc, 'packets/pubcomp/missed'), - {noreply, State} - end; + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + ensure_stats_timer(dequeue(acked(pubcomp, PacketId, State))); + false -> + ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubcomp/missed'), + State + end); %% RESUME: -handle_cast({resume, #{conn_pid := ConnPid, - will_msg := WillMsg, - expiry_interval := SessionExpiryInterval, - max_inflight := MaxInflight, - topic_alias_maximum := TopicAliasMaximum}}, State = #state{client_id = ClientId, - conn_pid = OldConnPid, - clean_start = CleanStart, - retry_timer = RetryTimer, - await_rel_timer = AwaitTimer, - expiry_timer = ExpireTimer, - will_delay_timer = WillDelayTimer}) -> +handle_cast({resume, #{conn_pid := ConnPid, + will_msg := WillMsg, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight}}, + State = #state{client_id = ClientId, + conn_pid = OldConnPid, + clean_start = CleanStart, + retry_timer = RetryTimer, + await_rel_timer = AwaitTimer, + expiry_timer = ExpireTimer, + will_delay_timer = WillDelayTimer}) -> ?LOG(info, "Resumed by connection ~p ", [ConnPid], State), %% Cancel Timers - lists:foreach(fun emqx_misc:cancel_timer/1, [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]), + lists:foreach(fun emqx_misc:cancel_timer/1, + [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]), case kick(ClientId, OldConnPid, ConnPid) of ok -> ?LOG(warning, "Connection ~p kickout ~p", [ConnPid, OldConnPid], State); @@ -542,19 +549,18 @@ handle_cast({resume, #{conn_pid := ConnPid, true = link(ConnPid), - State1 = State#state{conn_pid = ConnPid, - binding = binding(ConnPid), - old_conn_pid = OldConnPid, - clean_start = false, - retry_timer = undefined, - awaiting_rel = #{}, - await_rel_timer = undefined, - expiry_timer = undefined, - expiry_interval = SessionExpiryInterval, - inflight = emqx_inflight:update_size(MaxInflight, State#state.inflight), - topic_alias_maximum = TopicAliasMaximum, - will_delay_timer = undefined, - will_msg = WillMsg}, + State1 = State#state{conn_pid = ConnPid, + binding = binding(ConnPid), + old_conn_pid = OldConnPid, + clean_start = false, + retry_timer = undefined, + awaiting_rel = #{}, + await_rel_timer = undefined, + expiry_timer = undefined, + expiry_interval = ExpiryInterval, + inflight = emqx_inflight:update_size(MaxInflight, State#state.inflight), + will_delay_timer = undefined, + will_msg = WillMsg}, %% Clean Session: true -> false??? CleanStart andalso emqx_sm:set_session_attrs(ClientId, attrs(State1)), @@ -562,9 +568,9 @@ handle_cast({resume, #{conn_pid := ConnPid, emqx_hooks:run('session.resumed', [#{client_id => ClientId}, attrs(State)]), %% Replay delivery and Dequeue pending messages - noreply(dequeue(retry_delivery(true, State1))); + noreply(ensure_stats_timer(dequeue(retry_delivery(true, State1)))); -handle_cast({expiry_interval, Interval}, State) -> +handle_cast({update_expiry_interval, Interval}, State) -> {noreply, State#state{expiry_interval = Interval}}; handle_cast(Msg, State) -> @@ -573,9 +579,10 @@ handle_cast(Msg, State) -> %% Batch dispatch handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) -> - {noreply, lists:foldl(fun(Msg, NewState) -> - element(2, handle_info({dispatch, Topic, Msg}, NewState)) - end, State, Msgs)}; + noreply(lists:foldl( + fun(Msg, St) -> + element(2, handle_info({dispatch, Topic, Msg}, St)) + end, State, Msgs)); %% Dispatch message handle_info({dispatch, Topic, Msg = #message{}}, State) -> @@ -584,12 +591,11 @@ handle_info({dispatch, Topic, Msg = #message{}}, State) -> %% Require ack, but we do not have connection %% negative ack the message so it can try the next subscriber in the group ok = emqx_shared_sub:nack_no_connection(Msg), - noreply(State); + {noreply, State}; false -> - handle_dispatch(Topic, Msg, State) + noreply(ensure_stats_timer(handle_dispatch(Topic, Msg, State))) end; - %% Do nothing if the client has been disconnected. handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) -> noreply(State#state{retry_timer = undefined}); @@ -598,11 +604,13 @@ handle_info({timeout, Timer, retry_delivery}, State = #state{retry_timer = Timer noreply(retry_delivery(false, State#state{retry_timer = undefined})); handle_info({timeout, Timer, check_awaiting_rel}, State = #state{await_rel_timer = Timer}) -> - noreply(expire_awaiting_rel(State#state{await_rel_timer = undefined})); + State1 = State#state{await_rel_timer = undefined}, + noreply(ensure_stats_timer(expire_awaiting_rel(State1))); handle_info({timeout, Timer, emit_stats}, State = #state{client_id = ClientId, - stats_timer = Timer}) -> + stats_timer = Timer, + gc_state = GcState}) -> emqx_metrics:commit(), _ = emqx_sm:set_session_stats(ClientId, stats(State)), NewState = State#state{stats_timer = undefined}, @@ -611,20 +619,27 @@ handle_info({timeout, Timer, emit_stats}, continue -> {noreply, NewState}; hibernate -> - ok = emqx_gc:reset(), %% going to hibernate, reset gc stats - {noreply, NewState, hibernate}; + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; {shutdown, Reason} -> ?LOG(warning, "shutdown due to ~p", [Reason], NewState), shutdown(Reason, NewState) end; + handle_info({timeout, Timer, expired}, State = #state{expiry_timer = Timer}) -> - ?LOG(info, "expired, shutdown now:(", [], State), + ?LOG(info, "expired, shutdown now.", [], State), shutdown(expired, State); handle_info({timeout, Timer, will_delay}, State = #state{will_msg = WillMsg, will_delay_timer = Timer}) -> send_willmsg(WillMsg), {noreply, State#state{will_msg = undefined}}; +%% ConnPid is shutting down by the supervisor. +handle_info({'EXIT', ConnPid, Reason}, #state{conn_pid = ConnPid}) + when Reason =:= killed; Reason =:= shutdown -> + exit(Reason); + handle_info({'EXIT', ConnPid, Reason}, State = #state{will_msg = WillMsg, expiry_interval = 0, conn_pid = ConnPid}) -> send_willmsg(WillMsg), {stop, Reason, State#state{will_msg = undefined, conn_pid = undefined}}; @@ -641,47 +656,44 @@ handle_info({'EXIT', Pid, Reason}, State = #state{conn_pid = ConnPid}) -> ?LOG(error, "Unexpected EXIT: conn_pid=~p, exit_pid=~p, reason=~p", [ConnPid, Pid, Reason], State), {noreply, State}; + handle_info(Info, State) -> emqx_logger:error("[Session] unexpected info: ~p", [Info]), {noreply, State}. -terminate(Reason, #state{will_msg = WillMsg, client_id = ClientId, conn_pid = ConnPid}) -> - emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]), +terminate(Reason, #state{will_msg = WillMsg, + client_id = ClientId, + conn_pid = ConnPid, + old_conn_pid = OldConnPid}) -> send_willmsg(WillMsg), - %% Ensure to shutdown the connection - if - ConnPid =/= undefined -> - ConnPid ! {shutdown, Reason}; - true -> ok - end, - emqx_sm:unregister_session(ClientId). + [maybe_shutdown(Pid, Reason) || Pid <- [ConnPid, OldConnPid]], + emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]). code_change(_OldVsn, State, _Extra) -> {ok, State}. +maybe_shutdown(undefined, _Reason) -> + ok; +maybe_shutdown(Pid, normal) -> + Pid ! {shutdown, normal}; +maybe_shutdown(Pid, Reason) -> + exit(Pid, Reason). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ -has_connection(#state{conn_pid = Pid}) -> is_pid(Pid) andalso is_process_alive(Pid). +has_connection(#state{conn_pid = Pid}) -> + is_pid(Pid) andalso is_process_alive(Pid). -handle_dispatch(Topic, Msg = #message{headers = Headers}, - State = #state{subscriptions = SubMap, - topic_alias_maximum = TopicAliasMaximum - }) -> - TopicAlias = maps:get('Topic-Alias', Headers, undefined), - if - TopicAlias =:= undefined orelse TopicAlias =< TopicAliasMaximum -> - noreply(case maps:find(Topic, SubMap) of - {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); - {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); - error -> - dispatch(emqx_message:unset_flag(dup, Msg), State) - end); - true -> - noreply(State) +handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) -> + case maps:find(Topic, SubMap) of + {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); + {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); + error -> + dispatch(emqx_message:unset_flag(dup, Msg), State) end. suback(_From, undefined, _ReasonCodes) -> @@ -925,8 +937,7 @@ dequeue(State = #state{inflight = Inflight}) -> dequeue2(State = #state{mqueue = Q}) -> case emqx_mqueue:out(Q) of - {empty, _Q} -> - State; + {empty, _Q} -> State; {{value, Msg}, Q1} -> %% Dequeue more dequeue(dispatch(Msg, State#state{mqueue = Q1})) @@ -967,7 +978,8 @@ ensure_will_delay_timer(State = #state{will_msg = WillMsg}) -> send_willmsg(WillMsg), State#state{will_msg = undefined}. -ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined, +ensure_stats_timer(State = #state{enable_stats = true, + stats_timer = undefined, idle_timeout = IdleTimeout}) -> State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; ensure_stats_timer(State) -> @@ -986,9 +998,8 @@ next_pkt_id(State = #state{next_pkt_id = Id}) -> %% Inc stats inc_stats(deliver, Msg, State = #state{deliver_stats = I}) -> - MsgSize = msg_size(Msg), - ok = emqx_gc:inc(1, MsgSize), - State#state{deliver_stats = I + 1}; + State1 = maybe_gc({1, msg_size(Msg)}, State), + State1#state{deliver_stats = I + 1}; inc_stats(enqueue, _Msg, State = #state{enqueue_stats = I}) -> State#state{enqueue_stats = I + 1}. @@ -1005,10 +1016,17 @@ reply({Reply, State}) -> reply(Reply, State). reply(Reply, State) -> - {reply, Reply, ensure_stats_timer(State)}. + {reply, Reply, State}. noreply(State) -> - {noreply, ensure_stats_timer(State)}. + {noreply, State}. shutdown(Reason, State) -> - {stop, {shutdown, Reason}, State}. + {stop, Reason, State}. + +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}. + diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl index 644e33f37..efa64815b 100644 --- a/src/emqx_session_sup.erl +++ b/src/emqx_session_sup.erl @@ -14,31 +14,243 @@ -module(emqx_session_sup). --behavior(supervisor). +-behaviour(gen_server). --include("emqx.hrl"). +-export([start_link/1]). +-export([start_session/1, count_sessions/0]). --export([start_link/0, start_session/1]). +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). --export([init/1]). +-type(shutdown() :: brutal_kill | infinity | pos_integer()). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). +-record(state, { + sessions :: #{pid() => emqx_types:client_id()}, + mfargs :: mfa(), + shutdown :: shutdown(), + clean_down :: fun() + }). --spec(start_session(map()) -> {ok, pid()}). -start_session(Attrs) -> - supervisor:start_child(?MODULE, [Attrs]). +-define(SUP, ?MODULE). +-define(BATCH_EXIT, 100000). +-define(ERROR_MSG(Format, Args), + error_logger:error_msg("[~s] " ++ Format, [?MODULE | Args])). -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- +%% @doc Start session supervisor. +-spec(start_link(map()) -> emqx_types:startlink_ret()). +start_link(SessSpec) when is_map(SessSpec) -> + gen_server:start_link({local, ?SUP}, ?MODULE, [SessSpec], []). -init([]) -> - {ok, {{simple_one_for_one, 0, 1}, - [#{id => session, - start => {emqx_session, start_link, []}, - restart => temporary, - shutdown => 5000, - type => worker, - modules => [emqx_session]}]}}. +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Start a session. +-spec(start_session(map()) -> emqx_types:startlink_ret()). +start_session(SessAttrs) -> + gen_server:call(?SUP, {start_session, SessAttrs}, infinity). + +%% @doc Count sessions. +-spec(count_sessions() -> non_neg_integer()). +count_sessions() -> + gen_server:call(?SUP, count_sessions, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Spec]) -> + process_flag(trap_exit, true), + MFA = maps:get(start, Spec), + Shutdown = maps:get(shutdown, Spec, brutal_kill), + CleanDown = maps:get(clean_down, Spec, undefined), + State = #state{sessions = #{}, + mfargs = MFA, + shutdown = Shutdown, + clean_down = CleanDown + }, + {ok, State}. + +handle_call({start_session, SessAttrs = #{client_id := ClientId}}, _From, + State = #state{sessions = SessMap, mfargs = {M, F, Args}}) -> + try erlang:apply(M, F, [SessAttrs | Args]) of + {ok, Pid} -> + reply({ok, Pid}, State#state{sessions = maps:put(Pid, ClientId, SessMap)}); + ignore -> + reply(ignore, State); + {error, Reason} -> + reply({error, Reason}, State) + catch + _:Error:Stk -> + ?ERROR_MSG("Failed to start session ~p: ~p, stacktrace:~n~p", + [ClientId, Error, Stk]), + reply({error, Error}, State) + end; + +handle_call(count_sessions, _From, State = #state{sessions = SessMap}) -> + {reply, maps:size(SessMap), State}; + +handle_call(Req, _From, State) -> + ?ERROR_MSG("unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?ERROR_MSG("unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'EXIT', Pid, _Reason}, State = #state{sessions = SessMap, clean_down = CleanDown}) -> + SessPids = [Pid | drain_exit(?BATCH_EXIT, [])], + {SessItems, SessMap1} = erase_all(SessPids, SessMap), + (CleanDown =:= undefined) + orelse emqx_pool:async_submit( + fun lists:foreach/2, [CleanDown, SessItems]), + {noreply, State#state{sessions = SessMap1}}; + +handle_info(Info, State) -> + ?ERROR_MSG("unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, State) -> + terminate_children(State). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +drain_exit(0, Acc) -> + lists:reverse(Acc); +drain_exit(Cnt, Acc) -> + receive + {'EXIT', Pid, _Reason} -> + drain_exit(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + +erase_all(Pids, Map) -> + lists:foldl( + fun(Pid, {Acc, M}) -> + case maps:take(Pid, M) of + {Val, M1} -> + {[{Val, Pid}|Acc], M1}; + error -> + {Acc, M} + end + end, {[], Map}, Pids). + +terminate_children(State = #state{sessions = SessMap, shutdown = Shutdown}) -> + {Pids, EStack0} = monitor_children(SessMap), + Sz = sets:size(Pids), + EStack = + case Shutdown of + brutal_kill -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + infinity -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + Time when is_integer(Time) -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + TRef = erlang:start_timer(Time, self(), kill), + wait_children(Shutdown, Pids, Sz, TRef, EStack0) + end, + %% Unroll stacked errors and report them + dict:fold(fun(Reason, Pid, _) -> + report_error(connection_shutdown_error, Reason, Pid, State) + end, ok, EStack). + +monitor_children(SessMap) -> + lists:foldl( + fun(Pid, {Pids, EStack}) -> + case monitor_child(Pid) of + ok -> + {sets:add_element(Pid, Pids), EStack}; + {error, normal} -> + {Pids, EStack}; + {error, Reason} -> + {Pids, dict:append(Reason, Pid, EStack)} + end + end, {sets:new(), dict:new()}, maps:keys(SessMap)). + +%% Help function to shutdown/2 switches from link to monitor approach +monitor_child(Pid) -> + %% Do the monitor operation first so that if the child dies + %% before the monitoring is done causing a 'DOWN'-message with + %% reason noproc, we will get the real reason in the 'EXIT'-message + %% unless a naughty child has already done unlink... + erlang:monitor(process, Pid), + unlink(Pid), + + receive + %% If the child dies before the unlik we must empty + %% the mail-box of the 'EXIT'-message and the 'DOWN'-message. + {'EXIT', Pid, Reason} -> + receive + {'DOWN', _, process, Pid, _} -> + {error, Reason} + end + after 0 -> + %% If a naughty child did unlink and the child dies before + %% monitor the result will be that shutdown/2 receives a + %% 'DOWN'-message with reason noproc. + %% If the child should die after the unlink there + %% will be a 'DOWN'-message with a correct reason + %% that will be handled in shutdown/2. + ok + end. + +wait_children(_Shutdown, _Pids, 0, undefined, EStack) -> + EStack; +wait_children(_Shutdown, _Pids, 0, TRef, EStack) -> + %% If the timer has expired before its cancellation, we must empty the + %% mail-box of the 'timeout'-message. + erlang:cancel_timer(TRef), + receive + {timeout, TRef, kill} -> + EStack + after 0 -> + EStack + end; + +%%TODO: Copied from supervisor.erl, rewrite it later. +wait_children(brutal_kill, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, killed} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), + Sz-1, TRef, dict:append(Reason, Pid, EStack)) + end; + +wait_children(Shutdown, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, shutdown} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, normal} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, + TRef, dict:append(Reason, Pid, EStack)); + {timeout, TRef, kill} -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz-1, undefined, EStack) + end. + +report_error(Error, Reason, Pid, #state{mfargs = MFA}) -> + SupName = list_to_atom("esockd_connection_sup - " ++ pid_to_list(self())), + ErrorMsg = [{supervisor, SupName}, + {errorContext, Error}, + {reason, Reason}, + {offender, [{pid, Pid}, + {name, connection}, + {mfargs, MFA}]}], + error_logger:error_report(supervisor_report, ErrorMsg). + +reply(Repy, State) -> + {reply, Repy, State}. diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index ebe6d51f8..b7e41213b 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -17,6 +17,7 @@ -behaviour(gen_server). -include("emqx.hrl"). +-include("emqx_mqtt.hrl"). %% Mnesia bootstrap -export([mnesia/1]). @@ -27,7 +28,8 @@ -export([start_link/0]). -export([subscribe/3, unsubscribe/3]). --export([dispatch/3, maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]). +-export([dispatch/3]). +-export([maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]). %% for testing -export([subscribers/2]). @@ -38,6 +40,7 @@ -define(SERVER, ?MODULE). -define(TAB, emqx_shared_subscription). +-define(SHARED_SUBS, emqx_shared_subscriber). -define(ALIVE_SUBS, emqx_alive_shared_subscribers). -define(SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS, 5). -define(ack, shared_sub_ack). @@ -48,8 +51,6 @@ -record(state, {pmon}). -record(emqx_shared_subscription, {group, topic, subpid}). --include("emqx_mqtt.hrl"). - %%------------------------------------------------------------------------------ %% Mnesia bootstrap %%------------------------------------------------------------------------------ @@ -72,16 +73,11 @@ mnesia(copy) -> start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). -subscribe(undefined, _Topic, _SubPid) -> - ok; subscribe(Group, Topic, SubPid) when is_pid(SubPid) -> - mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), - gen_server:cast(?SERVER, {monitor, SubPid}). + gen_server:call(?SERVER, {subscribe, Group, Topic, SubPid}). -unsubscribe(undefined, _Topic, _SubPid) -> - ok; unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) -> - mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)). + gen_server:call(?SERVER, {unsubscribe, Group, Topic, SubPid}). record(Group, Topic, SubPid) -> #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}. @@ -251,14 +247,15 @@ do_pick_subscriber(Group, Topic, round_robin, _ClientId, Count) -> subscribers(Group, Topic) -> ets:select(?TAB, [{{emqx_shared_subscription, Group, Topic, '$1'}, [], ['$1']}]). -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%----------------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> - {atomic, PMon} = mnesia:transaction(fun init_monitors/0), mnesia:subscribe({table, ?TAB, simple}), - ets:new(?ALIVE_SUBS, [named_table, {read_concurrency, true}, protected]), + {atomic, PMon} = mnesia:transaction(fun init_monitors/0), + ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> @@ -267,14 +264,29 @@ init_monitors() -> emqx_pmon:monitor(SubPid, Mon) end, emqx_pmon:new(), ?TAB). +handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) -> + mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) + end, + ok = maybe_insert_alive_tab(SubPid), + true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), + {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; + +handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> + mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) + end, + {reply, ok, State}; + handle_call(Req, _From, State) -> emqx_logger:error("[SharedSub] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({monitor, SubPid}, State= #state{pmon = PMon}) -> - NewPmon = emqx_pmon:monitor(SubPid, PMon), - ok = maybe_insert_alive_tab(SubPid), - {noreply, update_stats(State#state{pmon = NewPmon})}; handle_cast(Msg, State) -> emqx_logger:error("[SharedSub] unexpected cast: ~p", [Msg]), {noreply, State}. @@ -316,12 +328,18 @@ maybe_insert_alive_tab(Pid) when is_pid(Pid) -> ets:insert(?ALIVE_SUBS, {Pid}), cleanup_down(SubPid) -> ?IS_LOCAL_PID(SubPid) orelse ets:delete(?ALIVE_SUBS, SubPid), lists:foreach( - fun(Record) -> - mnesia:dirty_delete_object(?TAB, Record) - end,mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). + fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) -> + ok = mnesia:dirty_delete_object(?TAB, Record), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) + end + end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). update_stats(State) -> - emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), State. + emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), + State. %% Return 'true' if the subscriber process is alive AND not in the failed list is_active_sub(Pid, FailedSubs) -> diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl index 8f9a3e3cb..ab270f1af 100644 --- a/src/emqx_sm.erl +++ b/src/emqx_sm.erl @@ -21,12 +21,15 @@ -export([start_link/0]). -export([open_session/1, close_session/1]). --export([lookup_session/1, lookup_session_pid/1]). -export([resume_session/2]). -export([discard_session/1, discard_session/2]). --export([register_session/2, unregister_session/1]). --export([get_session_attrs/1, set_session_attrs/2]). --export([get_session_stats/1, set_session_stats/2]). +-export([register_session/1, register_session/2]). +-export([unregister_session/1, unregister_session/2]). +-export([get_session_attrs/1, get_session_attrs/2, + set_session_attrs/2, set_session_attrs/3]). +-export([get_session_stats/1, get_session_stats/2, + set_session_stats/2, set_session_stats/3]). +-export([lookup_session_pids/1]). %% Internal functions for rpc -export([dispatch/3]). @@ -34,18 +37,23 @@ %% Internal function for stats -export([stats_fun/0]). +%% Internal function for emqx_session_sup +-export([clean_down/1]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(SM, ?MODULE). -%% ETS Tables --define(SESSION_TAB, emqx_session). --define(SESSION_P_TAB, emqx_persistent_session). +%% ETS Tables for session management. +-define(SESSION_TAB, emqx_session). +-define(SESSION_P_TAB, emqx_session_p). -define(SESSION_ATTRS_TAB, emqx_session_attrs). -define(SESSION_STATS_TAB, emqx_session_stats). +-define(BATCH_SIZE, 100000). + -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?SM}, ?MODULE, [], []). @@ -59,12 +67,11 @@ open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid end, emqx_sm_locker:trans(ClientId, CleanStart); -open_session(SessAttrs = #{clean_start := false, - client_id := ClientId}) -> +open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> ResumeStart = fun(_) -> case resume_session(ClientId, SessAttrs) of - {ok, SPid} -> - {ok, SPid, true}; + {ok, SessPid} -> + {ok, SessPid, true}; {error, not_found} -> emqx_session_sup:start_session(SessAttrs) end @@ -76,168 +83,162 @@ open_session(SessAttrs = #{clean_start := false, discard_session(ClientId) when is_binary(ClientId) -> discard_session(ClientId, self()). +-spec(discard_session(emqx_types:client_id(), pid()) -> ok). discard_session(ClientId, ConnPid) when is_binary(ClientId) -> - lists:foreach(fun({_ClientId, SPid}) -> - case catch emqx_session:discard(SPid, ConnPid) of - {Err, Reason} when Err =:= 'EXIT'; Err =:= error -> - emqx_logger:error("[SM] Failed to discard ~p: ~p", [SPid, Reason]); - ok -> ok - end - end, lookup_session(ClientId)). + lists:foreach( + fun(SessPid) -> + try emqx_session:discard(SessPid, ConnPid) + catch + _:Error:_Stk -> + emqx_logger:error("[SM] Failed to discard ~p: ~p", [SessPid, Error]) + end + end, lookup_session_pids(ClientId)). %% @doc Try to resume a session. -spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}). resume_session(ClientId, SessAttrs = #{conn_pid := ConnPid}) -> - case lookup_session(ClientId) of + case lookup_session_pids(ClientId) of [] -> {error, not_found}; - [{_ClientId, SPid}] -> - ok = emqx_session:resume(SPid, SessAttrs), - {ok, SPid}; - Sessions -> - [{_, SPid}|StaleSessions] = lists:reverse(Sessions), - emqx_logger:error("[SM] More than one session found: ~p", [Sessions]), - lists:foreach(fun({_, StalePid}) -> + [SessPid] -> + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid}; + SessPids -> + [SessPid|StalePids] = lists:reverse(SessPids), + emqx_logger:error("[SM] More than one session found: ~p", [SessPids]), + lists:foreach(fun(StalePid) -> catch emqx_session:discard(StalePid, ConnPid) - end, StaleSessions), - ok = emqx_session:resume(SPid, SessAttrs), - {ok, SPid} + end, StalePids), + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid} end. %% @doc Close a session. --spec(close_session({emqx_types:client_id(), pid()} | pid()) -> ok). -close_session({_ClientId, SPid}) -> - emqx_session:close(SPid); -close_session(SPid) when is_pid(SPid) -> - emqx_session:close(SPid). +-spec(close_session(emqx_types:client_id() | pid()) -> ok). +close_session(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> ok; + [SessPid] -> close_session(SessPid); + SessPids -> lists:foreach(fun close_session/1, SessPids) + end; -%% @doc Register a session with attributes. --spec(register_session(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - list(emqx_session:attr())) -> ok). -register_session(ClientId, SessAttrs) when is_binary(ClientId) -> - register_session({ClientId, self()}, SessAttrs); +close_session(SessPid) when is_pid(SessPid) -> + emqx_session:close(SessPid). -register_session(Session = {ClientId, SPid}, SessAttrs) - when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_TAB, Session), - ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), - proplists:get_value(clean_start, SessAttrs, true) - andalso ets:insert(?SESSION_P_TAB, Session), - emqx_sm_registry:register_session(Session), - notify({registered, ClientId, SPid}). +%% @doc Register a session. +-spec(register_session(emqx_types:client_id()) -> ok). +register_session(ClientId) when is_binary(ClientId) -> + register_session(ClientId, self()). -%% @doc Get session attrs --spec(get_session_attrs({emqx_types:client_id(), pid()}) -> list(emqx_session:attr())). -get_session_attrs(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_ATTRS_TAB, Session, []). - -%% @doc Set session attrs --spec(set_session_attrs(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - list(emqx_session:attr())) -> true). -set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) -> - set_session_attrs({ClientId, self()}, SessAttrs); -set_session_attrs(Session = {ClientId, SPid}, SessAttrs) when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}). +-spec(register_session(emqx_types:client_id(), pid()) -> ok). +register_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:insert(?SESSION_TAB, Session), + emqx_sm_registry:register_session(Session). %% @doc Unregister a session --spec(unregister_session(emqx_types:client_id() | {emqx_types:client_id(), pid()}) -> ok). +-spec(unregister_session(emqx_types:client_id()) -> ok). unregister_session(ClientId) when is_binary(ClientId) -> - unregister_session({ClientId, self()}); + unregister_session(ClientId, self()). -unregister_session(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - emqx_sm_registry:unregister_session(Session), - ets:delete(?SESSION_STATS_TAB, Session), - ets:delete(?SESSION_ATTRS_TAB, Session), - ets:delete_object(?SESSION_P_TAB, Session), - ets:delete_object(?SESSION_TAB, Session), - notify({unregistered, ClientId, SPid}). +-spec(unregister_session(emqx_types:client_id(), pid()) -> ok). +unregister_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:delete(?SESSION_STATS_TAB, Session), + true = ets:delete(?SESSION_ATTRS_TAB, Session), + true = ets:delete_object(?SESSION_P_TAB, Session), + true = ets:delete_object(?SESSION_TAB, Session), + emqx_sm_registry:unregister_session(Session). + +%% @doc Get session attrs +-spec(get_session_attrs(emqx_types:client_id()) -> list(emqx_session:attr())). +get_session_attrs(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> get_session_attrs(ClientId, SessPid) + end. + +-spec(get_session_attrs(emqx_types:client_id(), pid()) -> list(emqx_session:attr())). +get_session_attrs(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + emqx_tables:lookup_value(?SESSION_ATTRS_TAB, {ClientId, SessPid}, []). + +%% @doc Set session attrs +-spec(set_session_attrs(emqx_types:client_id(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) -> + set_session_attrs(ClientId, self(), SessAttrs). + +-spec(set_session_attrs(emqx_types:client_id(), pid(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessPid, SessAttrs) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), + proplists:get_value(clean_start, SessAttrs, true) orelse ets:insert(?SESSION_P_TAB, Session). %% @doc Get session stats --spec(get_session_stats({emqx_types:client_id(), pid()}) -> list(emqx_stats:stats())). -get_session_stats(Session = {ClientId, SPid}) when is_binary(ClientId), is_pid(SPid) -> - safe_lookup_element(?SESSION_STATS_TAB, Session, []). +-spec(get_session_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_session_stats(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> + get_session_stats(ClientId, SessPid) + end. + +-spec(get_session_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_session_stats(ClientId, SessPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?SESSION_STATS_TAB, {ClientId, SessPid}, []). %% @doc Set session stats --spec(set_session_stats(emqx_types:client_id() | {emqx_types:client_id(), pid()}, - emqx_stats:stats()) -> true). +-spec(set_session_stats(emqx_types:client_id(), emqx_stats:stats()) -> true). set_session_stats(ClientId, Stats) when is_binary(ClientId) -> - set_session_stats({ClientId, self()}, Stats); -set_session_stats(Session = {ClientId, SPid}, Stats) when is_binary(ClientId), is_pid(SPid) -> - ets:insert(?SESSION_STATS_TAB, {Session, Stats}). + set_session_stats(ClientId, self(), Stats). -%% @doc Lookup a session from registry --spec(lookup_session(emqx_types:client_id()) -> list({emqx_types:client_id(), pid()})). -lookup_session(ClientId) -> +-spec(set_session_stats(emqx_types:client_id(), pid(), emqx_stats:stats()) -> true). +set_session_stats(ClientId, SessPid, Stats) when is_binary(ClientId), is_pid(SessPid) -> + ets:insert(?SESSION_STATS_TAB, {{ClientId, SessPid}, Stats}). + +%% @doc Lookup session pid. +-spec(lookup_session_pids(emqx_types:client_id()) -> list(pid())). +lookup_session_pids(ClientId) -> case emqx_sm_registry:is_enabled() of - true -> emqx_sm_registry:lookup_session(ClientId); - false -> ets:lookup(?SESSION_TAB, ClientId) + true -> emqx_sm_registry:lookup_session(ClientId); + false -> emqx_tables:lookup_value(?SESSION_TAB, ClientId, []) end. %% @doc Dispatch a message to the session. -spec(dispatch(emqx_types:client_id(), emqx_topic:topic(), emqx_types:message()) -> any()). dispatch(ClientId, Topic, Msg) -> - case lookup_session_pid(ClientId) of - Pid when is_pid(Pid) -> - Pid ! {dispatch, Topic, Msg}; - undefined -> + case lookup_session_pids(ClientId) of + [SessPid|_] when is_pid(SessPid) -> + SessPid ! {dispatch, Topic, Msg}; + [] -> emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg]) end. -%% @doc Lookup session pid. --spec(lookup_session_pid(emqx_types:client_id()) -> pid() | undefined). -lookup_session_pid(ClientId) -> - safe_lookup_element(?SESSION_TAB, ClientId, undefined). - -safe_lookup_element(Tab, Key, Default) -> - try ets:lookup_element(Tab, Key, 2) - catch - error:badarg -> Default - end. - -notify(Event) -> - gen_server:cast(?SM, {notify, Event}). - %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ init([]) -> TabOpts = [public, set, {write_concurrency, true}], - _ = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]), - _ = emqx_tables:new(?SESSION_P_TAB, TabOpts), - _ = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), - _ = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), - emqx_stats:update_interval(sm_stats, fun ?MODULE:stats_fun/0), - {ok, #{session_pmon => emqx_pmon:new()}}. + ok = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), + ok = emqx_stats:update_interval(sess_stats, fun ?MODULE:stats_fun/0), + {ok, #{}}. handle_call(Req, _From, State) -> emqx_logger:error("[SM] unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast({notify, {registered, ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:monitor(SPid, ClientId, PMon)}}; - -handle_cast({notify, {unregistered, _ClientId, SPid}}, State = #{session_pmon := PMon}) -> - {noreply, State#{session_pmon := emqx_pmon:demonitor(SPid, PMon)}}; - handle_cast(Msg, State) -> emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #{session_pmon := PMon}) -> - case emqx_pmon:find(DownPid, PMon) of - undefined -> - {noreply, State}; - ClientId -> - unregister_session({ClientId, DownPid}), - {noreply, State#{session_pmon := emqx_pmon:erase(DownPid, PMon)}} - end; - handle_info(Info, State) -> emqx_logger:error("[SM] unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(sm_stats). + emqx_stats:cancel_update(sess_stats). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -246,6 +247,14 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ +clean_down(Session = {ClientId, SessPid}) -> + case ets:member(?SESSION_TAB, ClientId) + orelse ets:member(?SESSION_ATTRS_TAB, Session) of + true -> + unregister_session(ClientId, SessPid); + false -> ok + end. + stats_fun() -> safe_update_stats(?SESSION_TAB, 'sessions/count', 'sessions/max'), safe_update_stats(?SESSION_P_TAB, 'sessions/persistent/count', 'sessions/persistent/max'). diff --git a/src/emqx_sm_locker.erl b/src/emqx_sm_locker.erl index 29adf3342..409331b88 100644 --- a/src/emqx_sm_locker.erl +++ b/src/emqx_sm_locker.erl @@ -21,7 +21,7 @@ -export([trans/2, trans/3]). -export([lock/1, lock/2, unlock/1]). --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). +-spec(start_link() -> emqx_types:startlink_ret()). start_link() -> ekka_locker:start_link(?MODULE). diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl index b503d71c8..a7f44d771 100644 --- a/src/emqx_sm_registry.erl +++ b/src/emqx_sm_registry.erl @@ -20,7 +20,6 @@ -export([start_link/0]). -export([is_enabled/0]). - -export([register_session/1, lookup_session/1, unregister_session/1]). %% gen_server callbacks @@ -42,20 +41,25 @@ start_link() -> -spec(is_enabled() -> boolean()). is_enabled() -> - ets:info(?TAB, name) =/= undefined. + emqx_config:get_env(enable_session_registry, true). --spec(lookup_session(emqx_types:client_id()) - -> list({emqx_types:client_id(), session_pid()})). +-spec(lookup_session(emqx_types:client_id()) -> list(session_pid())). lookup_session(ClientId) -> - [{ClientId, SessPid} || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)]. + [SessPid || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)]. -spec(register_session({emqx_types:client_id(), session_pid()}) -> ok). register_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> - mnesia:dirty_write(?TAB, record(ClientId, SessPid)). + case is_enabled() of + true -> mnesia:dirty_write(?TAB, record(ClientId, SessPid)); + false -> ok + end. -spec(unregister_session({emqx_types:client_id(), session_pid()}) -> ok). unregister_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> - mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)). + case is_enabled() of + true -> mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)); + false -> ok + end. record(ClientId, SessPid) -> #global_session{sid = ClientId, pid = SessPid}. @@ -73,7 +77,7 @@ init([]) -> {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]), ok = ekka_mnesia:copy_table(?TAB), - _ = ekka:monitor(membership), + ok = ekka:monitor(membership), {ok, #{}}. handle_call(Req, _From, State) -> diff --git a/src/emqx_sm_sup.erl b/src/emqx_sm_sup.erl index 0be9facb0..ed491c67f 100644 --- a/src/emqx_sm_sup.erl +++ b/src/emqx_sm_sup.erl @@ -1,6 +1,5 @@ %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% -%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -26,25 +25,40 @@ start_link() -> init([]) -> %% Session locker - Locker = #{id => locker, - start => {emqx_sm_locker, start_link, []}, - restart => permanent, + Locker = #{id => locker, + start => {emqx_sm_locker, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm_locker]}, + type => worker, + modules => [emqx_sm_locker] + }, %% Session registry - Registry = #{id => registry, - start => {emqx_sm_registry, start_link, []}, - restart => permanent, + Registry = #{id => registry, + start => {emqx_sm_registry, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm_registry]}, + type => worker, + modules => [emqx_sm_registry] + }, %% Session Manager - Manager = #{id => manager, - start => {emqx_sm, start_link, []}, - restart => permanent, + Manager = #{id => manager, + start => {emqx_sm, start_link, []}, + restart => permanent, shutdown => 5000, - type => worker, - modules => [emqx_sm]}, - {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager]}}. + type => worker, + modules => [emqx_sm] + }, + %% Session Sup + SessSpec = #{start => {emqx_session, start_link, []}, + shutdown => brutal_kill, + clean_down => fun emqx_sm:clean_down/1 + }, + SessionSup = #{id => session_sup, + start => {emqx_session_sup, start_link, [SessSpec ]}, + restart => transient, + shutdown => infinity, + type => supervisor, + modules => [emqx_session_sup] + }, + {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager, SessionSup]}}. diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index 61ff6cbc3..790c397b9 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -152,7 +152,7 @@ cast(Msg) -> %%------------------------------------------------------------------------------ init(#{tick_ms := TickMs}) -> - _ = emqx_tables:new(?TAB, [set, public, {write_concurrency, true}]), + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([?CONNECTION_STATS, ?SESSION_STATS, ?PUBSUB_STATS, ?ROUTE_STATS, ?RETAINED_STATS]), true = ets:insert(?TAB, [{Name, 0} || Name <- Stats]), diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 2f29dbfee..60be4db87 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -69,8 +69,6 @@ init([]) -> AccessControl = worker_spec(emqx_access_control), %% Session Manager SMSup = supervisor_spec(emqx_sm_sup), - %% Session Sup - SessionSup = supervisor_spec(emqx_session_sup), %% Connection Manager CMSup = supervisor_spec(emqx_cm_sup), %% Sys Sup @@ -83,7 +81,6 @@ init([]) -> BridgeSup, AccessControl, SMSup, - SessionSup, CMSup, SysSup]}}. diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl index 330c87d9c..fdb106a99 100644 --- a/src/emqx_tables.erl +++ b/src/emqx_tables.erl @@ -15,12 +15,28 @@ -module(emqx_tables). -export([new/2]). +-export([lookup_value/2, lookup_value/3]). %% Create a named_table ets. +-spec(new(atom(), list()) -> ok). new(Tab, Opts) -> case ets:info(Tab, name) of undefined -> - ets:new(Tab, lists:usort([named_table | Opts])); - Tab -> Tab + _ = ets:new(Tab, lists:usort([named_table | Opts])), + ok; + Tab -> ok + end. + +%% KV lookup +-spec(lookup_value(atom(), term()) -> any()). +lookup_value(Tab, Key) -> + lookup_value(Tab, Key, undefined). + +-spec(lookup_value(atom(), term(), any()) -> any()). +lookup_value(Tab, Key, Def) -> + try + ets:lookup_element(Tab, Key, 2) + catch + error:badarg -> Def end. diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index 79f6042b7..27ff52827 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -36,7 +36,7 @@ %% @doc Create or replicate trie tables. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> - %% Optimize + %% Optimize storage StoreProps = [{ets, [{read_concurrency, true}, {write_concurrency, true}]}], %% Trie table @@ -72,7 +72,7 @@ insert(Topic) when is_binary(Topic) -> write_trie_node(TrieNode#trie_node{topic = Topic}); [] -> %% Add trie path - lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), + ok = lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), %% Add last node write_trie_node(#trie_node{node_id = Topic, topic = Topic}) end. @@ -93,7 +93,7 @@ lookup(NodeId) -> delete(Topic) when is_binary(Topic) -> case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{edge_count = 0}] -> - mnesia:delete({?TRIE_NODE, Topic}), + ok = mnesia:delete({?TRIE_NODE, Topic}), delete_path(lists:reverse(emqx_topic:triples(Topic))); [TrieNode] -> write_trie_node(TrieNode#trie_node{topic = undefined}); @@ -112,12 +112,12 @@ add_path({Node, Word, Child}) -> [TrieNode = #trie_node{edge_count = Count}] -> case mnesia:wread({?TRIE, Edge}) of [] -> - write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), + ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), write_trie(#trie{edge = Edge, node_id = Child}); [_] -> ok end; [] -> - write_trie_node(#trie_node{node_id = Node, edge_count = 1}), + ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}), write_trie(#trie{edge = Edge, node_id = Child}) end. @@ -154,10 +154,10 @@ match_node(NodeId, [W|Words], ResAcc) -> delete_path([]) -> ok; delete_path([{NodeId, Word, _} | RestPath]) -> - mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), - case mnesia:read(?TRIE_NODE, NodeId) of + ok = mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), + case mnesia:wread({?TRIE_NODE, NodeId}) of [#trie_node{edge_count = 1, topic = undefined}] -> - mnesia:delete({?TRIE_NODE, NodeId}), + ok = mnesia:delete({?TRIE_NODE, NodeId}), delete_path(RestPath); [TrieNode = #trie_node{edge_count = 1, topic = _}] -> write_trie_node(TrieNode#trie_node{edge_count = 0}); @@ -167,9 +167,11 @@ delete_path([{NodeId, Word, _} | RestPath]) -> mnesia:abort({node_not_found, NodeId}) end. +%% @private write_trie(Trie) -> mnesia:write(?TRIE, Trie, write). +%% @private write_trie_node(TrieNode) -> mnesia:write(?TRIE_NODE, TrieNode, write). diff --git a/src/emqx_vm.erl b/src/emqx_vm.erl index bf6388232..74b815795 100644 --- a/src/emqx_vm.erl +++ b/src/emqx_vm.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved. +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,7 +11,6 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -module(emqx_vm). diff --git a/src/emqx_zone.erl b/src/emqx_zone.erl index dd183dbdf..d119abe52 100644 --- a/src/emqx_zone.erl +++ b/src/emqx_zone.erl @@ -68,7 +68,7 @@ stop() -> %%------------------------------------------------------------------------------ init([]) -> - _ = emqx_tables:new(?TAB, [set, {read_concurrency, true}]), + ok = emqx_tables:new(?TAB, [set, {read_concurrency, true}]), {ok, element(2, handle_info(reload, #{timer => undefined}))}. handle_call(force_reload, _From, State) -> diff --git a/test/emqx_banned_SUITE.erl b/test/emqx_banned_SUITE.erl index 9d4c85134..7e7434e61 100644 --- a/test/emqx_banned_SUITE.erl +++ b/test/emqx_banned_SUITE.erl @@ -18,9 +18,7 @@ -compile(nowarn_export_all). -include("emqx.hrl"). - -include("emqx_mqtt.hrl"). - -include_lib("eunit/include/eunit.hrl"). all() -> [t_banned_all]. @@ -29,18 +27,27 @@ t_banned_all(_) -> emqx_ct_broker_helpers:run_setup_steps(), emqx_banned:start_link(), TimeNow = erlang:system_time(second), - Banned = #banned{who = {client_id, <<"TestClient">>}, + Banned = #banned{who = {client_id, <<"TestClient">>}, reason = <<"test">>, - by = <<"banned suite">>, - desc = <<"test">>, - until = TimeNow + 1}, + by = <<"banned suite">>, + desc = <<"test">>, + until = TimeNow + 1}, ok = emqx_banned:add(Banned), % here is not expire banned test because its check interval is greater than 5 mins, but its effect has been confirmed - ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), timer:sleep(2500), - ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), ok = emqx_banned:add(Banned), - ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), - emqx_banned:del({client_id, <<"TestClient">>}), - ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, username => undefined, peername => {undefined, undefined}})), + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + emqx_banned:delete({client_id, <<"TestClient">>}), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), emqx_ct_broker_helpers:run_teardown_steps(). + diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index f9a6dcf40..b2031a61c 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -36,7 +36,6 @@ groups() -> [ {pubsub, [sequence], [subscribe_unsubscribe, publish, pubsub, - t_local_subscribe, t_shared_subscribe, dispatch_with_no_sub, 'pubsub#', 'pubsub+']}, @@ -61,14 +60,14 @@ subscribe_unsubscribe(_) -> ok = emqx:subscribe(<<"topic">>, <<"clientId">>), ok = emqx:subscribe(<<"topic/1">>, <<"clientId">>, #{ qos => 1 }), ok = emqx:subscribe(<<"topic/2">>, <<"clientId">>, #{ qos => 2 }), - true = emqx:subscribed(<<"topic">>, <<"clientId">>), + true = emqx:subscribed(<<"clientId">>, <<"topic">>), Topics = emqx:topics(), lists:foreach(fun(Topic) -> ?assert(lists:member(Topic, Topics)) end, Topics), - ok = emqx:unsubscribe(<<"topic">>, <<"clientId">>), - ok = emqx:unsubscribe(<<"topic/1">>, <<"clientId">>), - ok = emqx:unsubscribe(<<"topic/2">>, <<"clientId">>). + ok = emqx:unsubscribe(<<"topic">>), + ok = emqx:unsubscribe(<<"topic/1">>), + ok = emqx:unsubscribe(<<"topic/2">>). publish(_) -> Msg = emqx_message:make(ct, <<"test/pubsub">>, <<"hello">>), @@ -85,18 +84,25 @@ dispatch_with_no_sub(_) -> pubsub(_) -> true = emqx:is_running(node()), Self = self(), - Subscriber = {Self, <<"clientId">>}, - ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 1 }), - #{qos := 1} = ets:lookup_element(emqx_suboption, {<<"a/b/c">>, Subscriber}, 2), - #{qos := 1} = emqx:get_subopts(<<"a/b/c">>, Subscriber), - true = emqx:set_subopts(<<"a/b/c">>, Subscriber, #{qos => 0}), - #{qos := 0} = emqx:get_subopts(<<"a/b/c">>, Subscriber), - ok = emqx:subscribe(<<"a/b/c">>, <<"clientId">>, #{ qos => 2 }), + Subscriber = <<"clientId">>, + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 1 }), + #{qos := 1} = ets:lookup_element(emqx_suboption, {Self, <<"a/b/c">>}, 2), + #{qos := 1} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + true = emqx_broker:set_subopts(<<"a/b/c">>, #{qos => 0}), + #{qos := 0} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 2 }), %% ct:log("Emq Sub: ~p.~n", [ets:lookup(emqx_suboption, {<<"a/b/c">>, Subscriber})]), timer:sleep(10), - [{Self, <<"clientId">>}] = emqx_broker:subscribers(<<"a/b/c">>), + [Self] = emqx_broker:subscribers(<<"a/b/c">>), emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)), - ?assert(receive {dispatch, <<"a/b/c">>, _ } -> true; P -> ct:log("Receive Message: ~p~n",[P]) after 2 -> false end), + ?assert( + receive {dispatch, <<"a/b/c">>, _ } -> + true; + P -> + ct:log("Receive Message: ~p~n",[P]) + after 2 -> + false + end), spawn(fun() -> emqx:subscribe(<<"a/b/c">>), emqx:subscribe(<<"c/d/e">>), @@ -106,38 +112,15 @@ pubsub(_) -> timer:sleep(20), emqx:unsubscribe(<<"a/b/c">>). -t_local_subscribe(_) -> - ok = emqx:subscribe(<<"$local/topic0">>), - ok = emqx:subscribe(<<"$local/topic1">>, <<"clientId">>), - ok = emqx:subscribe(<<"$local/topic2">>, <<"clientId">>, #{ qos => 2 }), - timer:sleep(10), - ?assertEqual([{self(), undefined}], emqx:subscribers("$local/topic0")), - ?assertEqual([{self(), <<"clientId">>}], emqx:subscribers("$local/topic1")), - ?assertEqual([{<<"$local/topic1">>, #{ qos => 0 }}, - {<<"$local/topic2">>, #{ qos => 2 }}], - emqx:subscriptions({self(), <<"clientId">>})), - ?assertEqual(ok, emqx:unsubscribe("$local/topic0")), - ?assertEqual(ok, emqx:unsubscribe("$local/topic0")), - ?assertEqual(ok, emqx:unsubscribe("$local/topic1", <<"clientId">>)), - ?assertEqual(ok, emqx:unsubscribe("$local/topic2", <<"clientId">>)), - ?assertEqual([], emqx:subscribers("topic1")), - ?assertEqual([], emqx:subscriptions({self(), <<"clientId">>})). - t_shared_subscribe(_) -> - emqx:subscribe("$local/$share/group1/topic1"), emqx:subscribe("$share/group2/topic2"), emqx:subscribe("$queue/topic3"), timer:sleep(10), - ct:log("share subscriptions: ~p~n", [emqx:subscriptions({self(), undefined})]), - ?assertEqual([{self(), undefined}], emqx:subscribers(<<"$local/$share/group1/topic1">>)), - ?assertEqual([{<<"$local/$share/group1/topic1">>, #{qos => 0}}, - {<<"$queue/topic3">>, #{qos => 0}}, - {<<"$share/group2/topic2">>, #{qos => 0}}], - lists:sort(emqx:subscriptions({self(), undefined}))), - emqx:unsubscribe("$local/$share/group1/topic1"), + ct:log("share subscriptions: ~p~n", [emqx:subscriptions(self())]), + ?assertEqual(2, length(emqx:subscriptions(self()))), emqx:unsubscribe("$share/group2/topic2"), emqx:unsubscribe("$queue/topic3"), - ?assertEqual([], lists:sort(emqx:subscriptions(self()))). + ?assertEqual(0, length(emqx:subscriptions(self()))). 'pubsub#'(_) -> emqx:subscribe(<<"a/#">>), diff --git a/test/emqx_client_SUITE.erl b/test/emqx_client_SUITE.erl index 021109606..13303bfdc 100644 --- a/test/emqx_client_SUITE.erl +++ b/test/emqx_client_SUITE.erl @@ -32,9 +32,8 @@ <<"+/+">>, <<"TopicA/#">>]). all() -> - [ {group, mqttv4}, - {group, mqttv5} - ]. + [{group, mqttv4}, + {group, mqttv5}]. groups() -> [{mqttv4, [non_parallel_tests], @@ -48,8 +47,7 @@ groups() -> dollar_topics_test]}, {mqttv5, [non_parallel_tests], [request_response, - share_sub_request_topic]} -]. + share_sub_request_topic]}]. init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index 5e29e075e..3d879a476 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -1,3 +1,4 @@ + %% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,21 +18,53 @@ -compile(export_all). -compile(nowarn_export_all). +-include("emqx.hrl"). -include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). -all() -> [t_register_unregister_connection]. +all() -> [{group, cm}]. -t_register_unregister_connection(_) -> - {ok, _} = emqx_cm_sup:start_link(), - Pid = self(), - emqx_cm:register_connection(<<"conn1">>), - emqx_cm:register_connection({<<"conn2">>, Pid}, [{port, 8080}, {ip, "192.168.0.1"}]), - timer:sleep(2000), - [{<<"conn1">>, Pid}] = emqx_cm:lookup_connection(<<"conn1">>), - [{<<"conn2">>, Pid}] = emqx_cm:lookup_connection(<<"conn2">>), - Pid = emqx_cm:lookup_conn_pid(<<"conn1">>), - emqx_cm:unregister_connection(<<"conn1">>), - [] = emqx_cm:lookup_connection(<<"conn1">>), - [{port, 8080}, {ip, "192.168.0.1"}] = emqx_cm:get_conn_attrs({<<"conn2">>, Pid}), - emqx_cm:set_conn_stats(<<"conn2">>, [[{count, 1}, {max, 2}]]), - [[{count, 1}, {max, 2}]] = emqx_cm:get_conn_stats({<<"conn2">>, Pid}). +groups() -> + [{cm, [non_parallel_tests], + [t_get_set_conn_attrs, + t_get_set_conn_stats, + t_lookup_conn_pid]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +init_per_testcase(_TestCase, Config) -> + register_connection(), + Config. + +end_per_testcase(_TestCase, _Config) -> + unregister_connection(), + ok. + +t_get_set_conn_attrs(_) -> + ?assert(emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}])), + ?assert(emqx_cm:set_conn_attrs(<<"conn2">>, self(), [{port, 8080}, {ip, "192.168.0.2"}])), + ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs(<<"conn1">>)), + ?assertEqual([{port, 8080}, {ip, "192.168.0.2"}], emqx_cm:get_conn_attrs(<<"conn2">>, self())). + +t_get_set_conn_stats(_) -> + ?assert(emqx_cm:set_conn_stats(<<"conn1">>, [{count, 1}, {max, 2}])), + ?assert(emqx_cm:set_conn_stats(<<"conn2">>, self(), [{count, 1}, {max, 2}])), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn1">>)), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn2">>, self())). + +t_lookup_conn_pid(_) -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>, self())), + ?assertEqual(self(), emqx_cm:lookup_conn_pid(<<"conn1">>)). + +register_connection() -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:register_connection(<<"conn2">>, self())). + +unregister_connection() -> + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn2">>, self())). diff --git a/test/emqx_gc_SUITE.erl b/test/emqx_gc_SUITE.erl new file mode 100644 index 000000000..22d7cd584 --- /dev/null +++ b/test/emqx_gc_SUITE.erl @@ -0,0 +1,57 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_init, t_run, t_info, t_reset]. + +t_init(_) -> + ?assertEqual(undefined, emqx_gc:init(false)), + GC1 = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC1)), + GC2 = emqx_gc:init(#{count => 0, bytes => 10}), + ?assertEqual(#{oct => {10, 10}}, emqx_gc:info(GC2)), + GC3 = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual(#{cnt => {10, 10}, oct => {10, 10}}, emqx_gc:info(GC3)). + +t_run(_) -> + GC = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual({true, GC}, emqx_gc:run(1, 1000, GC)), + ?assertEqual({true, GC}, emqx_gc:run(1000, 1, GC)), + {false, GC1} = emqx_gc:run(1, 1, GC), + ?assertEqual(#{cnt => {10, 9}, oct => {10, 9}}, emqx_gc:info(GC1)), + {false, GC2} = emqx_gc:run(2, 2, GC1), + ?assertEqual(#{cnt => {10, 7}, oct => {10, 7}}, emqx_gc:info(GC2)), + {false, GC3} = emqx_gc:run(3, 3, GC2), + ?assertEqual(#{cnt => {10, 4}, oct => {10, 4}}, emqx_gc:info(GC3)), + ?assertEqual({true, GC}, emqx_gc:run(4, 4, GC3)). + +t_info(_) -> + ?assertEqual(undefined, emqx_gc:info(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC)). + +t_reset(_) -> + ?assertEqual(undefined, emqx_gc:reset(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 10}), + {false, GC1} = emqx_gc:run(5, 5, GC), + ?assertEqual(#{cnt => {10, 5}, oct => {10, 5}}, emqx_gc:info(GC1)), + ?assertEqual(GC, emqx_gc:reset(GC1)). + diff --git a/test/emqx_gc_tests.erl b/test/emqx_gc_tests.erl deleted file mode 100644 index ffcac91d1..000000000 --- a/test/emqx_gc_tests.erl +++ /dev/null @@ -1,53 +0,0 @@ -%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_gc_tests). - --include_lib("eunit/include/eunit.hrl"). - -trigger_by_cnt_test() -> - Args = #{count => 2, bytes => 0}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1000), - St1 = inspect(), - ?assertMatch({_, Remain} when Remain > 0, maps:get(cnt, St1)), - ok = emqx_gc:inc(2, 2), - St2 = inspect(), - ok = emqx_gc:inc(0, 2000), - St3 = inspect(), - ?assertEqual(St2, St3), - ?assertMatch({N, N}, maps:get(cnt, St2)), - ?assertNot(maps:is_key(oct, St2)), - ok. - -trigger_by_oct_test() -> - Args = #{count => 2, bytes => 2}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1), - St1 = inspect(), - ?assertMatch({_, Remain} when Remain > 0, maps:get(oct, St1)), - ok = emqx_gc:inc(2, 2), - St2 = inspect(), - ?assertMatch({N, N}, maps:get(oct, St2)), - ?assertMatch({M, M}, maps:get(cnt, St2)), - ok. - -disabled_test() -> - Args = #{count => -1, bytes => false}, - ok = emqx_gc:init(Args), - ok = emqx_gc:inc(1, 1), - ?assertEqual(#{}, inspect()), - ok. - -inspect() -> erlang:get(emqx_gc). diff --git a/test/emqx_pd_SUITE.erl b/test/emqx_pd_SUITE.erl new file mode 100644 index 000000000..e53fa7539 --- /dev/null +++ b/test/emqx_pd_SUITE.erl @@ -0,0 +1,31 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pd_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> [update_counter]. + +update_counter(_) -> + ?assertEqual(undefined, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(1, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(2, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(3, emqx_pd:get_counter(bytes)), + ?assertEqual(3, emqx_pd:reset_counter(bytes)), + ?assertEqual(0, emqx_pd:get_counter(bytes)). + diff --git a/test/emqx_pmon_SUITE.erl b/test/emqx_pmon_SUITE.erl new file mode 100644 index 000000000..67e8cf4d7 --- /dev/null +++ b/test/emqx_pmon_SUITE.erl @@ -0,0 +1,48 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pmon_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_monitor, t_find, t_erase]. + +t_monitor(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + PMon2 = emqx_pmon:demonitor(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)). + +t_find(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + ?assertEqual({ok, val}, emqx_pmon:find(self(), PMon1)), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(error, emqx_pmon:find(self(), PMon2)). + +t_erase(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)), + {Items, PMon3} = emqx_pmon:erase_all([self()], PMon1), + ?assertEqual([{self(), val}], Items), + ?assertEqual(0, emqx_pmon:count(PMon3)). + diff --git a/test/emqx_protocol_SUITE.erl b/test/emqx_protocol_SUITE.erl index 38d1001d3..3bf9976c7 100644 --- a/test/emqx_protocol_SUITE.erl +++ b/test/emqx_protocol_SUITE.erl @@ -62,6 +62,7 @@ init_per_suite(Config) -> {App, SchemaFile, ConfigFile} <- [{emqx, deps_path(emqx, "priv/emqx.schema"), deps_path(emqx, "etc/emqx.conf")}]], + emqx_zone:set_env(external, max_topic_alias, 20), Config. end_per_suite(_Config) -> @@ -162,6 +163,82 @@ connect_v5(_) -> raw_recv_parse(Data, ?MQTT_PROTO_V5) end), + % topic alias = 0 + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 0}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5) + end), + + % topic alias maximum + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 15}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data3} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBACK_PACKET(1, 0), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5), + + {ok, Data4} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, _, <<"hello">>), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 2, #{'Topic-Alias' => 21}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data5} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data5, ?MQTT_PROTO_V5) + end), + % test clean start with_connection(fun([Sock]) -> emqx_client_sock:send(Sock, diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl index e317ec7b3..c115fd0cd 100644 --- a/test/emqx_router_SUITE.erl +++ b/test/emqx_router_SUITE.erl @@ -21,17 +21,16 @@ -compile(nowarn_export_all). -define(R, emqx_router). --define(TABS, [emqx_route, emqx_trie, emqx_trie_node]). all() -> [{group, route}]. groups() -> [{route, [sequence], - [add_del_route, - match_routes, - has_routes, - router_add_del]}]. + [t_add_delete, + t_do_add_delete, + t_match_routes, + t_has_routes]}]. init_per_suite(Config) -> emqx_ct_broker_helpers:run_setup_steps(), @@ -47,77 +46,47 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). -add_del_route(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/b/c">>, node()), - timer:sleep(1), - - ?R:add_route(From, <<"a/+/b">>, node()), - ct:log("Topics: ~p ~n", [emqx_topic:wildcard(<<"a/+/b">>)]), - timer:sleep(1), - +t_add_delete(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/b">>, node()), ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), - ?R:del_route(From, <<"a/b/c">>, node()), + ?R:delete_route(<<"a/b/c">>), + ?R:delete_route(<<"a/+/b">>, node()), + ?assertEqual([], ?R:topics()). - ?R:del_route(From, <<"a/+/b">>, node()), - timer:sleep(120), - ?assertEqual([], lists:sort(?R:topics())). +t_do_add_delete(_) -> + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/+/b">>, node()), + ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), -match_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"a/b/c">>, node()), - ?R:add_route(From, <<"a/+/c">>, node()), - ?R:add_route(From, <<"a/b/#">>, node()), - ?R:add_route(From, <<"#">>, node()), - timer:sleep(1000), + ?R:do_delete_route(<<"a/b/c">>, node()), + ?R:do_delete_route(<<"a/+/b">>), + ?assertEqual([], ?R:topics()). + +t_match_routes(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/c">>, node()), + ?R:add_route(<<"a/b/#">>, node()), + ?R:add_route(<<"#">>, node()), ?assertEqual([#route{topic = <<"#">>, dest = node()}, #route{topic = <<"a/+/c">>, dest = node()}, #route{topic = <<"a/b/#">>, dest = node()}, #route{topic = <<"a/b/c">>, dest = node()}], - lists:sort(?R:match_routes(<<"a/b/c">>))). + lists:sort(?R:match_routes(<<"a/b/c">>))), + ?R:delete_route(<<"a/b/c">>, node()), + ?R:delete_route(<<"a/+/c">>, node()), + ?R:delete_route(<<"a/b/#">>, node()), + ?R:delete_route(<<"#">>, node()), + ?assertEqual([], lists:sort(?R:match_routes(<<"a/b/c">>))). -has_routes(_) -> - From = {self(), make_ref()}, - ?R:add_route(From, <<"devices/+/messages">>, node()), - timer:sleep(200), - ?assert(?R:has_routes(<<"devices/+/messages">>)). +t_has_routes(_) -> + ?R:add_route(<<"devices/+/messages">>, node()), + ?assert(?R:has_routes(<<"devices/+/messages">>)), + ?R:delete_route(<<"devices/+/messages">>). clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, ?TABS). - -router_add_del(_) -> - ?R:add_route(<<"#">>), - ?R:add_route(<<"a/b/c">>, node()), - ?R:add_route(<<"+/#">>), - Routes = [R1, R2 | _] = [ - #route{topic = <<"#">>, dest = node()}, - #route{topic = <<"+/#">>, dest = node()}, - #route{topic = <<"a/b/c">>, dest = node()}], - timer:sleep(500), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - ?R:print_routes(<<"a/b/c">>), - - %% Batch Add - lists:foreach(fun(R) -> ?R:add_route(R) end, Routes), - ?assertEqual(Routes, lists:sort(?R:match_routes(<<"a/b/c">>))), - - %% Del - ?R:del_route(<<"a/b/c">>, node()), - timer:sleep(500), - [R1, R2] = lists:sort(?R:match_routes(<<"a/b/c">>)), - {atomic, []} = mnesia:transaction(fun emqx_trie:lookup/1, [<<"a/b/c">>]), - - %% Batch Del - R3 = #route{topic = <<"#">>, dest = 'a@127.0.0.1'}, - ?R:add_route(R3), - ?R:del_route(<<"#">>), - ?R:del_route(R2), - ?R:del_route(R3), - timer:sleep(500), - [] = lists:sort(?R:match_routes(<<"a/b/c">>)). + lists:foreach(fun mnesia:clear_table/1, [emqx_route, emqx_trie, emqx_trie_node]). diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl new file mode 100644 index 000000000..ab408b8e0 --- /dev/null +++ b/test/emqx_sequence_SUITE.erl @@ -0,0 +1,38 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-import(emqx_sequence, [nextval/2, reclaim/2]). + +all() -> + [sequence_generate]. + +sequence_generate(_) -> + ok = emqx_sequence:create(seqtab), + ?assertEqual(1, nextval(seqtab, key)), + ?assertEqual(2, nextval(seqtab, key)), + ?assertEqual(3, nextval(seqtab, key)), + ?assertEqual(2, reclaim(seqtab, key)), + ?assertEqual(1, reclaim(seqtab, key)), + ?assertEqual(0, reclaim(seqtab, key)), + ?assertEqual(false, ets:member(seqtab, key)), + ?assertEqual(1, nextval(seqtab, key)), + ?assert(emqx_sequence:delete(seqtab)). + diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index a04a0b82b..b8e0aedd3 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -53,21 +53,10 @@ t_session_all(_) -> emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 2}}]), emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 1}}]), timer:sleep(200), - [{<<"topic">>, _}] = emqx:subscriptions({SPid, <<"ClientId">>}), + [{<<"topic">>, _}] = emqx:subscriptions(SPid), emqx_session:publish(SPid, 1, Message1), timer:sleep(200), {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid), - emqx_session:puback(SPid, 2), - emqx_session:puback(SPid, 3, reasoncode), - emqx_session:pubrec(SPid, 4), - emqx_session:pubrec(SPid, 5, reasoncode), - emqx_session:pubrel(SPid, 6, reasoncode), - emqx_session:pubcomp(SPid, 7, reasoncode), - timer:sleep(200), - 2 = emqx_metrics:val('packets/puback/missed'), - 2 = emqx_metrics:val('packets/pubrec/missed'), - 1 = emqx_metrics:val('packets/pubrel/missed'), - 1 = emqx_metrics:val('packets/pubcomp/missed'), Attrs = emqx_session:attrs(SPid), Info = emqx_session:info(SPid), Stats = emqx_session:stats(SPid), @@ -76,5 +65,5 @@ t_session_all(_) -> 1 = proplists:get_value(subscriptions_count, Stats), emqx_session:unsubscribe(SPid, [<<"topic">>]), timer:sleep(200), - [] = emqx:subscriptions({SPid, <<"clientId">>}), + [] = emqx:subscriptions(SPid), emqx_mock_client:close_session(ConnPid). diff --git a/test/emqx_sm_SUITE.erl b/test/emqx_sm_SUITE.erl index 3aed3090e..b3ce70c82 100644 --- a/test/emqx_sm_SUITE.erl +++ b/test/emqx_sm_SUITE.erl @@ -15,35 +15,78 @@ -module(emqx_sm_SUITE). -include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). -compile(export_all). -compile(nowarn_export_all). -all() -> [t_open_close_session]. +-define(ATTRS, #{clean_start => true, + client_id => <<"client">>, + zone => internal, + username => <<"emqx">>, + expiry_interval => 0, + max_inflight => 0, + topic_alias_maximum => 0, + will_msg => undefined}). + +all() -> [{group, sm}]. + +groups() -> + [{sm, [non_parallel_tests], + [t_open_close_session, + t_resume_session, + t_discard_session, + t_register_unregister_session, + t_get_set_session_attrs, + t_get_set_session_stats, + t_lookup_session_pids]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). t_open_close_session(_) -> - emqx_ct_broker_helpers:run_setup_steps(), {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), - Attrs = #{clean_start => true, - client_id => <<"client">>, - conn_pid => ClientPid, - zone => internal, - username => <<"emqx">>, - expiry_interval => 0, - max_inflight => 0, - topic_alias_maximum => 0, - will_msg => undefined}, - {ok, SPid} = emqx_sm:open_session(Attrs), - [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>), - SPid = emqx_sm:lookup_session_pid(<<"client">>), - {ok, NewConnPid} = emqx_mock_client:start_link(<<"client">>), - {ok, SPid, true} = emqx_sm:open_session(Attrs#{clean_start => false, conn_pid => NewConnPid}), - [{<<"client">>, SPid}] = emqx_sm:lookup_session(<<"client">>), - SAttrs = emqx_sm:get_session_attrs({<<"client">>, SPid}), - <<"client">> = proplists:get_value(client_id, SAttrs), - Session = {<<"client">>, SPid}, - emqx_sm:set_session_stats(Session, {open, true}), - {open, true} = emqx_sm:get_session_stats(Session), - ok = emqx_sm:close_session(SPid), - [] = emqx_sm:lookup_session(<<"client">>), - emqx_ct_broker_helpers:run_teardown_steps(). + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:close_session(SPid)). + +t_resume_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual({ok, SPid}, emqx_sm:resume_session(<<"client">>, ?ATTRS#{conn_pid => ClientPid})). + +t_discard_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client1">>), + {ok, _SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:discard_session(<<"client1">>)). + +t_register_unregister_session(_) -> + Pid = self(), + {ok, _ClientPid} = emqx_mock_client:start_link(<<"client">>), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>, Pid)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>), Pid). + +t_get_set_session_attrs(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, [?ATTRS#{conn_pid => ClientPid}])), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, SPid, [?ATTRS#{conn_pid => ClientPid}])), + [SAttr] = emqx_sm:get_session_attrs(<<"client">>, SPid), + ?assertEqual(<<"client">>, maps:get(client_id, SAttr)). + +t_get_set_session_stats(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, [{inflight, 10}])), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}])), + ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)). + +t_lookup_session_pids(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)). diff --git a/test/emqx_stats_tests.erl b/test/emqx_stats_tests.erl index dd9733a88..e8b5e82af 100644 --- a/test/emqx_stats_tests.erl +++ b/test/emqx_stats_tests.erl @@ -75,10 +75,10 @@ helper_test_() -> with_proc(fun() -> TestF(CbModule, CbFun) end, TickMs) end end, - [{"emqx_broker_helper", MkTestFun(emqx_broker_helper, stats_fun)}, + [{"emqx_broker", MkTestFun(emqx_broker, stats_fun)}, {"emqx_sm", MkTestFun(emqx_sm, stats_fun)}, {"emqx_router_helper", MkTestFun(emqx_router_helper, stats_fun)}, - {"emqx_cm", MkTestFun(emqx_cm, update_conn_stats)} + {"emqx_cm", MkTestFun(emqx_cm, stats_fun)} ]. with_proc(F) -> diff --git a/test/emqx_tables_SUITE.erl b/test/emqx_tables_SUITE.erl index 95590b0e9..1002c0a0b 100644 --- a/test/emqx_tables_SUITE.erl +++ b/test/emqx_tables_SUITE.erl @@ -20,7 +20,7 @@ all() -> [t_new]. t_new(_) -> - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - ets:insert(TId, {loss, 100}), - TId = emqx_tables:new(test_table, [{read_concurrency, true}]), - 100 = ets:lookup_element(TId, loss, 2). + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + ets:insert(test_table, {key, 100}), + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + 100 = ets:lookup_element(test_table, key, 2). diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index 85637a447..500fe3574 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -24,7 +24,7 @@ -define(TRIE_TABS, [emqx_trie, emqx_trie_node]). all() -> - [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. + [t_mnesia, t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. init_per_suite(Config) -> application:load(emqx), @@ -42,41 +42,44 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> clear_tables(). +t_mnesia(_) -> + ok = ?TRIE:mnesia(copy). + t_insert(_) -> TN = #trie_node{node_id = <<"sensor">>, edge_count = 3, topic = <<"sensor">>, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:lookup(<<"sensor">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:lookup(<<"sensor">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_match(_) -> Machted = [<<"sensor/+/#">>, <<"sensor/#">>], - {atomic, Machted} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:match(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:match(<<"sensor/1">>) + end, + ?assertEqual({atomic, Machted}, mnesia:transaction(Fun)). t_match2(_) -> Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, - {atomic, Matched} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"#">>), - ?TRIE:insert(<<"+/#">>), - ?TRIE:insert(<<"+/+/#">>), - {?TRIE:match(<<"a/b/c">>), - ?TRIE:match(<<"$SYS/broker/zenmq">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"#">>), + ?TRIE:insert(<<"+/#">>), + ?TRIE:insert(<<"+/+/#">>), + {?TRIE:match(<<"a/b/c">>), + ?TRIE:match(<<"$SYS/broker/zenmq">>)} + end, + ?assertEqual({atomic, Matched}, mnesia:transaction(Fun)). t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], @@ -91,43 +94,42 @@ t_delete(_) -> edge_count = 2, topic = undefined, flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/#">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:lookup(<<"sensor/1">>) - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/#">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:lookup(<<"sensor/1">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). t_delete2(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric/3">>), - {?TRIE:lookup(<<"sensor">>), - ?TRIE:lookup(<<"sensor/1">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric/3">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). t_delete3(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/+">>), - ?TRIE:insert(<<"sensor/+/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor/+/metric/2">>), - ?TRIE:delete(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/+">>), - ?TRIE:delete(<<"sensor/+/unknown">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} - end). + Fun = fun() -> + ?TRIE:insert(<<"sensor/+">>), + ?TRIE:insert(<<"sensor/+/metric/2">>), + ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor/+/metric/2">>), + ?TRIE:delete(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/+">>), + ?TRIE:delete(<<"sensor/+/unknown">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). clear_tables() -> lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS).