diff --git a/.gitignore b/.gitignore index 7a4e891d1..ab0cbe156 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ cuttlefish rebar.lock xrefr erlang.mk +*.coverdata diff --git a/Makefile b/Makefile index 6022dbf8f..fe94d309d 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,14 @@ ERLC_OPTS += +debug_info -DAPPLICATION=emqx BUILD_DEPS = cuttlefish dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1 -#TEST_DEPS = emqx_ct_helplers -#dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers +TEST_DEPS = meck +dep_meck = hex-emqx 0.8.13 TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx EUNIT_OPTS = verbose -# CT_SUITES = emqx_frame +# CT_SUITES = emqx_bridge ## emqx_trie emqx_router emqx_frame emqx_mqtt_compat CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ @@ -37,7 +37,8 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ - emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message + emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message emqx_os_mon \ + emqx_vm_mon emqx_alarm_handler CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) @@ -96,17 +97,24 @@ rebar-deps: @rebar3 get-deps rebar-eunit: $(CUTTLEFISH_SCRIPT) - @rebar3 eunit + @rebar3 eunit -v rebar-compile: @rebar3 compile -rebar-ct: app.config +rebar-ct-setup: app.config @rebar3 as test compile @ln -s -f '../../../../etc' _build/test/lib/emqx/ @ln -s -f '../../../../data' _build/test/lib/emqx/ + +rebar-ct: rebar-ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') +## Run one single CT with rebar3 +## e.g. make ct-one-suite suite=emqx_bridge +ct-one-suite: rebar-ct-setup + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE + rebar-clean: @rebar3 clean diff --git a/etc/emqx.conf b/etc/emqx.conf index 166fca25c..5361ab7da 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1596,28 +1596,6 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ##-------------------------------------------------------------------- ## Bridges to aws ##-------------------------------------------------------------------- -## Start type of the bridge. -## -## Value: enum -## manual -## auto -## bridge.aws.start_type = manual - -## Bridge reconnect time. -## -## Value: Duration -## Default: 30 seconds -## bridge.aws.reconnect_interval = 30s - -## Retry interval for bridge QoS1 message delivering. -## -## Value: Duration -## bridge.aws.retry_interval = 20s - -## Inflight size. -## -## Value: Integer -## bridge.aws.max_inflight = 32 ## Bridge address: node name for local bridge, host:port for remote. ## @@ -1662,66 +1640,12 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: String ## bridge.aws.mountpoint = bridge/aws/${node}/ -## Ping interval of a down bridge. -## -## Value: Duration -## Default: 10 seconds -## bridge.aws.keepalive = 60s - ## Forward message topics ## ## Value: String ## Example: topic1/#,topic2/# ## bridge.aws.forwards = topic1/#,topic2/# -## Subscriptions of the bridge topic. -## -## Value: String -## bridge.aws.subscription.1.topic = cmd/topic1 - -## Subscriptions of the bridge qos. -## -## Value: Number -## bridge.aws.subscription.1.qos = 1 - -## Subscriptions of the bridge topic. -## -## Value: String -## bridge.aws.subscription.2.topic = cmd/topic2 - -## Subscriptions of the bridge qos. -## -## Value: Number -## bridge.aws.subscription.2.qos = 1 - -## If enabled, queue would be written into disk more quickly. -## However, If disabled, some message would be dropped in -## the situation emqx crashed. -## -## Value: on | off -## bridge.aws.queue.mem_cache = on - -## Batch size for buffer queue stored -## -## Value: Integer -## default: 1000 -## bridge.aws.queue.batch_size = 1000 - -## Base directory for replayq to store messages on disk -## If this config entry is missing or set to undefined, -## replayq works in a mem-only manner. If the config -## entry was set to `bridge.aws.mqueue_type = memory` -## this config entry would have no effect on mqueue -## -## Value: String -## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ - -## Replayq segment size -## -## Value: Bytesize - -## bridge.aws.queue.replayq_seg_bytes = 10MB - ## Bribge to remote server via SSL. ## ## Value: on | off @@ -1747,36 +1671,89 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: String ## bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +## bridge.aws.keepalive = 60s + ## TLS versions used by the bridge. ## ## Value: String ## bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 -##-------------------------------------------------------------------- -## Bridges to azure -##-------------------------------------------------------------------- +## Subscriptions of the bridge topic. +## +## Value: String +## bridge.aws.subscription.1.topic = cmd/topic1 + +## Subscriptions of the bridge qos. +## +## Value: Number +## bridge.aws.subscription.1.qos = 1 + +## Subscriptions of the bridge topic. +## +## Value: String +## bridge.aws.subscription.2.topic = cmd/topic2 + +## Subscriptions of the bridge qos. +## +## Value: Number +## bridge.aws.subscription.2.qos = 1 + ## Start type of the bridge. ## ## Value: enum ## manual ## auto -## bridge.azure.start_type = manual - -## Bridge reconnect count. -## -## Value: Number -## bridge.azure.reconnect_count = 10 +## bridge.aws.start_type = manual ## Bridge reconnect time. ## ## Value: Duration ## Default: 30 seconds -## bridge.azure.reconnect_time = 30s +## bridge.aws.reconnect_interval = 30s ## Retry interval for bridge QoS1 message delivering. ## ## Value: Duration -## bridge.azure.retry_interval = 20s +## bridge.aws.retry_interval = 20s + +## Inflight size. +## +## Value: Integer +## bridge.aws.max_inflight_batches = 32 + +## Max number of messages to collect in a batch for +## each send call towards emqx_bridge_connect +## +## Value: Integer +## default: 32 +## bridge.aws.queue.batch_count_limit = 32 + +## Max number of bytes to collect in a batch for each +## send call towards emqx_bridge_connect +## +## Value: Bytesize +## default: 1000M +## bridge.aws.queue.batch_bytes_limit = 1000MB + +## Base directory for replayq to store messages on disk +## If this config entry is missing or set to undefined, +## replayq works in a mem-only manner. +## +## Value: String +## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ + +## Replayq segment size +## +## Value: Bytesize +## bridge.aws.queue.replayq_seg_bytes = 10MB + +##-------------------------------------------------------------------- +## Bridges to azure +##-------------------------------------------------------------------- ## Bridge address: node name for local bridge, host:port for remote. ## @@ -1819,13 +1796,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Mountpoint of the bridge. ## ## Value: String -## bridge.azure.mountpoint = bridge/azure/${node}/ - -## Ping interval of a down bridge. -## -## Value: Duration -## Default: 10 seconds -## bridge.azure.keepalive = 10s +## bridge.azure.mountpoint = bridge/aws/${node}/ ## Forward message topics ## @@ -1833,10 +1804,46 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Example: topic1/#,topic2/# ## bridge.azure.forwards = topic1/#,topic2/# +## Bribge to remote server via SSL. +## +## Value: on | off +## bridge.azure.ssl = off + +## PEM-encoded CA certificates of the bridge. +## +## Value: File +## bridge.azure.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem + +## Client SSL Certfile of the bridge. +## +## Value: File +## bridge.azure.certfile = {{ platform_etc_dir }}/certs/client-cert.pem + +## Client SSL Keyfile of the bridge. +## +## Value: File +## bridge.azure.keyfile = {{ platform_etc_dir }}/certs/client-key.pem + +## SSL Ciphers used by the bridge. +## +## Value: String +## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +## bridge.azure.keepalive = 60s + +## TLS versions used by the bridge. +## +## Value: String +## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1 + ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.azure.subscription.1.topic = $share/cmd/topic1 +## bridge.azure.subscription.1.topic = cmd/topic1 ## Subscriptions of the bridge qos. ## @@ -1846,27 +1853,50 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.azure.subscription.2.topic = $share/cmd/topic2 +## bridge.azure.subscription.2.topic = cmd/topic2 ## Subscriptions of the bridge qos. ## ## Value: Number ## bridge.azure.subscription.2.qos = 1 -## Batch size for buffer queue stored +## Start type of the bridge. +## +## Value: enum +## manual +## auto +## bridge.azure.start_type = manual + +## Bridge reconnect time. +## +## Value: Duration +## Default: 30 seconds +## bridge.azure.reconnect_interval = 30s + +## Retry interval for bridge QoS1 message delivering. +## +## Value: Duration +## bridge.azure.retry_interval = 20s + +## Inflight size. ## ## Value: Integer -## default: 1000 -## bridge.azure.queue.batch_size = 1000 +## bridge.azure.max_inflight_batches = 32 + +## Maximum number of messages in one batch when sending to remote borkers +## NOTE: when bridging via MQTT connection to remote broker, this config is only +## used for internal message passing optimization as the underlying MQTT +## protocol does not supports batching. +## +## Value: Integer +## default: 32 +## bridge.azure.queue.batch_size = 32 ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, -## replayq works in a mem-only manner. If the config -## entry was set to `bridge.aws.mqueue_type = memory` -## this config entry would have no effect on mqueue +## replayq works in a mem-only manner. ## ## Value: String -## Default: {{ platform_data_dir }}/emqx_aws_bridge/ ## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ ## Replayq segment size @@ -1874,30 +1904,6 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: Bytesize ## bridge.azure.queue.replayq_seg_bytes = 10MB -## PEM-encoded CA certificates of the bridge. -## -## Value: File -## bridge.azure.cacertfile = cacert.pem - -## Client SSL Certfile of the bridge. -## -## Value: File -## bridge.azure.certfile = cert.pem - -## Client SSL Keyfile of the bridge. -## -## Value: File -## bridge.azure.keyfile = key.pem - -## SSL Ciphers used by the bridge. -## -## Value: String -## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 - -## TLS versions used by the bridge. -## -## Value: String -## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1 ##-------------------------------------------------------------------- ## Modules @@ -2049,4 +2055,61 @@ sysmon.busy_port = false ## Value: true | false sysmon.busy_dist_port = true +## The time interval for the periodic cpu check +## +## Value: Duration +## -h: hour, e.g. '2h' for 2 hours +## -m: minute, e.g. '5m' for 5 minutes +## -s: second, e.g. '30s' for 30 seconds +## +## Default: 60s +os_mon.cpu_check_interval = 60s + +## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. +## +## Default: 80% +os_mon.cpu_high_watermark = 80% + +## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. +## +## Default: 60% +os_mon.cpu_low_watermark = 60% + +## The time interval for the periodic memory check +## +## Value: Duration +## -h: hour, e.g. '2h' for 2 hours +## -m: minute, e.g. '5m' for 5 minutes +## -s: second, e.g. '30s' for 30 seconds +## +## Default: 60s +os_mon.mem_check_interval = 60s + +## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. +## +## Default: 70% +os_mon.sysmem_high_watermark = 70% + +## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. +## +## Default: 5% +os_mon.procmem_high_watermark = 5% + +## The time interval for the periodic process limit check +## +## Value: Duration +## +## Default: 30s +vm_mon.check_interval = 30s + +## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. +## +## Default: 80% +vm_mon.process_high_watermark = 80% + +## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. +## +## Default: 60% +vm_mon.process_low_watermark = 60% + {{ additional_configs }} diff --git a/src/emqx_local_bridge_sup.erl b/include/emqx_client.hrl similarity index 63% rename from src/emqx_local_bridge_sup.erl rename to include/emqx_client.hrl index db349b94d..535b8ad55 100644 --- a/src/emqx_local_bridge_sup.erl +++ b/include/emqx_client.hrl @@ -12,15 +12,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_local_bridge_sup). - --include("emqx.hrl"). - --export([start_link/3]). - --spec(start_link(node(), emqx_topic:topic(), [emqx_local_bridge:option()]) - -> {ok, pid()} | {error, term()}). -start_link(Node, Topic, Options) -> - MFA = {emqx_local_bridge, start_link, [Node, Topic, Options]}, - emqx_pool_sup:start_link({bridge, Node, Topic}, random, MFA). +-ifndef(EMQX_CLIENT_HRL). +-define(EMQX_CLIENT_HRL, true). +-record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, + packet_id, topic, props, payload}). +-endif. diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 7e7670112..1c2ce1a27 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -171,9 +171,10 @@ -define(RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED, 16#A2). %%-------------------------------------------------------------------- -%% Maximum MQTT Packet Length +%% Maximum MQTT Packet ID and Length %%-------------------------------------------------------------------- +-define(MAX_PACKET_ID, 16#ffff). -define(MAX_PACKET_SIZE, 16#fffffff). %%-------------------------------------------------------------------- diff --git a/include/logger.hrl b/include/logger.hrl index f87668cc7..503b6cf21 100644 --- a/include/logger.hrl +++ b/include/logger.hrl @@ -35,6 +35,8 @@ -define(ALERT(Format), ?LOG(alert, Format, [])). -define(ALERT(Format, Args), ?LOG(alert, Format, Args)). +-define(LOG(Level, Format), ?LOG(Level, Format, [])). + -define(LOG(Level, Format, Args), begin (logger:log(Level,#{},#{report_cb => fun(_) -> {(Format), (Args)} end})) diff --git a/priv/emqx.schema b/priv/emqx.schema index 9e71248a7..5161d5e94 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1512,22 +1512,6 @@ end}. %%-------------------------------------------------------------------- %% Bridges %%-------------------------------------------------------------------- -{mapping, "bridge.$name.queue.mem_cache", "emqx.bridges", [ - {datatype, flag} -]}. - -{mapping, "bridge.$name.queue.batch_size", "emqx.bridges", [ - {datatype, integer} -]}. - -{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [ - {datatype, bytesize} -]}. - {mapping, "bridge.$name.address", "emqx.bridges", [ {datatype, string} ]}. @@ -1616,11 +1600,27 @@ end}. {datatype, {duration, ms}} ]}. -{mapping, "bridge.$name.max_inflight", "emqx.bridges", [ +{mapping, "bridge.$name.max_inflight_batches", "emqx.bridges", [ {default, 0}, {datatype, integer} ]}. +{mapping, "bridge.$name.queue.batch_count_limit", "emqx.bridges", [ + {datatype, integer} +]}. + +{mapping, "bridge.$name.queue.batch_bytes_limit", "emqx.bridges", [ + {datatype, bytesize} +]}. + +{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [ + {datatype, bytesize} +]}. + {translation, "emqx.bridges", fun(Conf) -> Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, @@ -1661,17 +1661,67 @@ end}. lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, "subscription", I, "topic"], Topic} <- Configs])], [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, "subscription", I, "qos"], QoS} <- Configs])]) end, - - maps:to_list( - lists:foldl( + IsNodeAddr = fun(Addr) -> + case string:tokens(Addr, "@") of + [_NodeName, _Hostname] -> true; + _ -> false + end + end, + ConnMod = fun(Name) -> + [AddrConfig] = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".address", Conf), + {_, Addr} = AddrConfig, + Subs = Subscriptions(Name), + case IsNodeAddr(Addr) of + true when Subs =/= [] -> + error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs}); + true -> + emqx_bridge_rpc; + false -> + emqx_bridge_mqtt + end + end, + %% to be backward compatible + Translate = + fun Tr(queue, Q, Cfg) -> + NewQ = maps:fold(Tr, #{}, Q), + Cfg#{queue => NewQ}; + Tr(address, Addr0, Cfg) -> + Addr = case IsNodeAddr(Addr0) of + true -> list_to_atom(Addr0); + false -> Addr0 + end, + Cfg#{address => Addr}; + Tr(batch_size, Count, Cfg) -> + Cfg#{batch_count_limit => Count}; + Tr(reconnect_interval, Ms, Cfg) -> + Cfg#{reconnect_delay_ms => Ms}; + Tr(max_inflight, Count, Cfg) -> + Cfg#{max_inflight_batches => Count}; + Tr(proto_ver, Ver, Cfg) -> + Cfg#{proto_ver => + case Ver of + mqttv3 -> v3; + mqttv4 -> v4; + mqttv5 -> v5; + _ -> v4 + end}; + Tr(Key, Value, Cfg) -> + Cfg#{Key => Value} + end, + C = lists:foldl( fun({["bridge", Name, Opt], Val}, Acc) -> %% e.g #{aws => [{OptKey, OptVal}]} - Init = [{list_to_atom(Opt), Val},{subscriptions, Subscriptions(Name)}, {queue, Queue(Name)}], - maps:update_with(list_to_atom(Name), - fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc); + Init = [{list_to_atom(Opt), Val}, + {connect_module, ConnMod(Name)}, + {subscriptions, Subscriptions(Name)}, + {queue, Queue(Name)}], + maps:update_with(list_to_atom(Name), fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc); (_, Acc) -> Acc - end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf)))) - + end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf))), + C1 = maps:map(fun(Bn, Bc) -> + maps:to_list(maps:fold(Translate, #{}, maps:from_list(Bc))) + end, C), + maps:to_list(C1) end}. %%-------------------------------------------------------------------- @@ -1844,3 +1894,70 @@ end}. {busy_port, cuttlefish:conf_get("sysmon.busy_port", Conf)}, {busy_dist_port, cuttlefish:conf_get("sysmon.busy_dist_port", Conf)}] end}. + +%%-------------------------------------------------------------------- +%% Operating System Monitor +%%-------------------------------------------------------------------- + +{mapping, "os_mon.cpu_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.cpu_high_watermark", "emqx.os_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.cpu_low_watermark", "emqx.os_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.mem_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.sysmem_high_watermark", "emqx.os_mon", [ + {default, "70%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.procmem_high_watermark", "emqx.os_mon", [ + {default, "5%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.os_mon", fun(Conf) -> + [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)}, + {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf)}, + {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf)}, + {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)}, + {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf)}, + {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf)}] +end}. + +%%-------------------------------------------------------------------- +%% VM Monitor +%%-------------------------------------------------------------------- +{mapping, "vm_mon.check_interval", "emqx.vm_mon", [ + {default, 30}, + {datatype, {duration, s}} +]}. + +{mapping, "vm_mon.process_high_watermark", "emqx.vm_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "vm_mon.process_low_watermark", "emqx.vm_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.vm_mon", fun(Conf) -> + [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)}, + {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf)}, + {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf)}] +end}. diff --git a/rebar.config b/rebar.config index 7486e7267..745c22f68 100644 --- a/rebar.config +++ b/rebar.config @@ -27,3 +27,5 @@ {cover_export_enabled, true}. {plugins, [coveralls]}. + +{profiles, [{test, [{deps, [{meck, "0.8.13"}]}]}]}. diff --git a/src/emqx.app.src b/src/emqx.app.src index f9c174ae3..c88be22c7 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -4,7 +4,7 @@ {modules,[]}, {registered,[emqx_sup]}, {applications,[kernel,stdlib,jsx,gproc,gen_rpc,esockd,cowboy, - replayq]}, + replayq,sasl,os_mon]}, {env,[]}, {mod,{emqx_app,[]}}, {maintainers,["Feng Lee "]}, diff --git a/src/emqx_alarm_handler.erl b/src/emqx_alarm_handler.erl new file mode 100644 index 000000000..8a9260ab9 --- /dev/null +++ b/src/emqx_alarm_handler.erl @@ -0,0 +1,169 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_alarm_handler). + +-behaviour(gen_event). + +-include("emqx.hrl"). +-include("logger.hrl"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([init/1, + handle_event/2, + handle_call/2, + handle_info/2, + terminate/2]). + +-export([load/0, + get_alarms/0]). + +-record(common_alarm, {id, desc}). +-record(alarm_history, {id, clear_at}). + +-define(ALARM_TAB, emqx_alarm). +-define(ALARM_HISTORY_TAB, emqx_alarm_history). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?ALARM_TAB, [ + {type, set}, + {disc_copies, [node()]}, + {local_content, true}, + {record_name, common_alarm}, + {attributes, record_info(fields, common_alarm)}]), + ok = ekka_mnesia:create_table(?ALARM_HISTORY_TAB, [ + {type, set}, + {disc_copies, [node()]}, + {local_content, true}, + {record_name, alarm_history}, + {attributes, record_info(fields, alarm_history)}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?ALARM_TAB), + ok = ekka_mnesia:copy_table(?ALARM_HISTORY_TAB). + +%%---------------------------------------------------------------------- +%% API +%%---------------------------------------------------------------------- + +load() -> + gen_event:swap_handler(alarm_handler, {alarm_handler, swap}, {?MODULE, []}). + +get_alarms() -> + gen_event:call(alarm_handler, ?MODULE, get_alarms). + +%%---------------------------------------------------------------------- +%% gen_event callbacks +%%---------------------------------------------------------------------- + +init({_Args, {alarm_handler, ExistingAlarms}}) -> + init_tables(ExistingAlarms), + {ok, []}; +init(_) -> + init_tables([]), + {ok, []}. + +handle_event({set_alarm, {AlarmId, AlarmDesc = #alarm{timestamp = undefined}}}, State) -> + handle_event({set_alarm, {AlarmId, AlarmDesc#alarm{timestamp = os:timestamp()}}}, State); +handle_event({set_alarm, Alarm = {AlarmId, AlarmDesc}}, State) -> + ?LOG(notice, "Alarm report: set ~p", [Alarm]), + case encode_alarm(Alarm) of + {ok, Json} -> + emqx_broker:safe_publish(alarm_msg(topic(alert, maybe_to_binary(AlarmId)), Json)); + {error, Reason} -> + ?LOG(error, "Failed to encode alarm: ~p", [Reason]) + end, + set_alarm_(AlarmId, AlarmDesc), + {ok, State}; +handle_event({clear_alarm, AlarmId}, State) -> + ?LOG(notice, "Alarm report: clear ~p", [AlarmId]), + emqx_broker:safe_publish(alarm_msg(topic(clear, maybe_to_binary(AlarmId)), <<"">>)), + clear_alarm_(AlarmId), + {ok, State}; +handle_event(_, State) -> + {ok, State}. + +handle_info(_, State) -> {ok, State}. + +handle_call(get_alarms, State) -> + {ok, get_alarms_(), State}; +handle_call(_Query, State) -> {ok, {error, bad_query}, State}. + +terminate(swap, _State) -> + {emqx_alarm_handler, get_alarms_()}; +terminate(_, _) -> + ok. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +init_tables(ExistingAlarms) -> + mnesia:clear_table(?ALARM_TAB), + lists:foreach(fun({Id, _Desc}) -> + set_alarm_history(Id) + end, ExistingAlarms). + +encode_alarm({AlarmId, #alarm{severity = Severity, + title = Title, + summary = Summary, + timestamp = Ts}}) -> + emqx_json:safe_encode([{id, maybe_to_binary(AlarmId)}, + {desc, [{severity, Severity}, + {title, iolist_to_binary(Title)}, + {summary, iolist_to_binary(Summary)}, + {ts, emqx_time:now_secs(Ts)}]}]); +encode_alarm({AlarmId, AlarmDesc}) -> + emqx_json:safe_encode([{id, maybe_to_binary(AlarmId)}, + {desc, maybe_to_binary(AlarmDesc)}]). + +alarm_msg(Topic, Payload) -> + Msg = emqx_message:make(?MODULE, Topic, Payload), + emqx_message:set_headers(#{'Content-Type' => <<"application/json">>}, + emqx_message:set_flag(sys, Msg)). + +topic(alert, AlarmId) -> + emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>); +topic(clear, AlarmId) -> + emqx_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>). + +maybe_to_binary(Data) when is_binary(Data) -> + Data; +maybe_to_binary(Data) -> + iolist_to_binary(io_lib:format("~p", [Data])). + +set_alarm_(Id, Desc) -> + mnesia:dirty_write(?ALARM_TAB, #common_alarm{id = Id, desc = Desc}). + +clear_alarm_(Id) -> + mnesia:dirty_delete(?ALARM_TAB, Id), + set_alarm_history(Id). + +get_alarms_() -> + Alarms = ets:tab2list(?ALARM_TAB), + [{Id, Desc} || #common_alarm{id = Id, desc = Desc} <- Alarms]. + +set_alarm_history(Id) -> + mnesia:dirty_write(?ALARM_HISTORY_TAB, #alarm_history{id = Id, + clear_at = undefined}). + + diff --git a/src/emqx_alarm_mgr.erl b/src/emqx_alarm_mgr.erl deleted file mode 100644 index cfb99678e..000000000 --- a/src/emqx_alarm_mgr.erl +++ /dev/null @@ -1,144 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_alarm_mgr). - --behaviour(gen_event). - --include("emqx.hrl"). --include("logger.hrl"). - --export([start_link/0]). --export([alarm_fun/0, get_alarms/0, set_alarm/1, clear_alarm/1]). --export([add_alarm_handler/1, add_alarm_handler/2, delete_alarm_handler/1]). - -%% gen_event callbacks --export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, - code_change/3]). - --define(ALARM_MGR, ?MODULE). - -start_link() -> - start_with( - fun(Pid) -> - gen_event:add_handler(Pid, ?MODULE, []) - end). - -start_with(Fun) -> - case gen_event:start_link({local, ?ALARM_MGR}) of - {ok, Pid} -> Fun(Pid), {ok, Pid}; - Error -> Error - end. - -alarm_fun() -> alarm_fun(false). - -alarm_fun(Bool) -> - fun(alert, _Alarm) when Bool =:= true -> alarm_fun(true); - (alert, Alarm) when Bool =:= false -> set_alarm(Alarm), alarm_fun(true); - (clear, AlarmId) when Bool =:= true -> clear_alarm(AlarmId), alarm_fun(false); - (clear, _AlarmId) when Bool =:= false -> alarm_fun(false) - end. - --spec(set_alarm(emqx_types:alarm()) -> ok). -set_alarm(Alarm) when is_record(Alarm, alarm) -> - gen_event:notify(?ALARM_MGR, {set_alarm, Alarm}). - --spec(clear_alarm(any()) -> ok). -clear_alarm(AlarmId) when is_binary(AlarmId) -> - gen_event:notify(?ALARM_MGR, {clear_alarm, AlarmId}). - --spec(get_alarms() -> list(emqx_types:alarm())). -get_alarms() -> - gen_event:call(?ALARM_MGR, ?MODULE, get_alarms). - -add_alarm_handler(Module) when is_atom(Module) -> - gen_event:add_handler(?ALARM_MGR, Module, []). - -add_alarm_handler(Module, Args) when is_atom(Module) -> - gen_event:add_handler(?ALARM_MGR, Module, Args). - -delete_alarm_handler(Module) when is_atom(Module) -> - gen_event:delete_handler(?ALARM_MGR, Module, []). - -%%------------------------------------------------------------------------------ -%% Default Alarm handler -%%------------------------------------------------------------------------------ - -init(_) -> {ok, #{alarms => []}}. - -handle_event({set_alarm, Alarm = #alarm{timestamp = undefined}}, State)-> - handle_event({set_alarm, Alarm#alarm{timestamp = os:timestamp()}}, State); - -handle_event({set_alarm, Alarm = #alarm{id = AlarmId}}, State = #{alarms := Alarms}) -> - case encode_alarm(Alarm) of - {ok, Json} -> - emqx_broker:safe_publish(alarm_msg(alert, AlarmId, Json)); - {error, Reason} -> - ?ERROR("[AlarmMgr] Failed to encode alarm: ~p", [Reason]) - end, - {ok, State#{alarms := [Alarm|Alarms]}}; - -handle_event({clear_alarm, AlarmId}, State = #{alarms := Alarms}) -> - case emqx_json:safe_encode([{id, AlarmId}, {ts, os:system_time(second)}]) of - {ok, Json} -> - emqx_broker:safe_publish(alarm_msg(clear, AlarmId, Json)); - {error, Reason} -> - ?ERROR("[AlarmMgr] Failed to encode clear: ~p", [Reason]) - end, - {ok, State#{alarms := lists:keydelete(AlarmId, 2, Alarms)}, hibernate}; - -handle_event(Event, State)-> - ?ERROR("[AlarmMgr] unexpected event: ~p", [Event]), - {ok, State}. - -handle_info(Info, State) -> - ?ERROR("[AlarmMgr] unexpected info: ~p", [Info]), - {ok, State}. - -handle_call(get_alarms, State = #{alarms := Alarms}) -> - {ok, Alarms, State}; - -handle_call(Req, State) -> - ?ERROR("[AlarmMgr] unexpected call: ~p", [Req]), - {ok, ignored, State}. - -terminate(swap, State) -> - {?MODULE, State}; -terminate(_, _) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -encode_alarm(#alarm{id = AlarmId, severity = Severity, title = Title, - summary = Summary, timestamp = Ts}) -> - emqx_json:safe_encode([{id, AlarmId}, {severity, Severity}, - {title, iolist_to_binary(Title)}, - {summary, iolist_to_binary(Summary)}, - {ts, emqx_time:now_secs(Ts)}]). - -alarm_msg(Type, AlarmId, Json) -> - Msg = emqx_message:make(?ALARM_MGR, topic(Type, AlarmId), Json), - emqx_message:set_headers( #{'Content-Type' => <<"application/json">>}, - emqx_message:set_flag(sys, Msg)). - -topic(alert, AlarmId) -> - emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>); -topic(clear, AlarmId) -> - emqx_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>). - diff --git a/src/emqx_app.erl b/src/emqx_app.erl index 30483dbde..e1583e128 100644 --- a/src/emqx_app.erl +++ b/src/emqx_app.erl @@ -40,6 +40,10 @@ start(_Type, _Args) -> emqx_listeners:start(), start_autocluster(), register(emqx, self()), + + emqx_alarm_handler:load(), + emqx_logger_handler:init(), + print_vsn(), {ok, Sup}. diff --git a/src/emqx_bridge.erl b/src/emqx_bridge.erl index 1ee5612e6..af85df68e 100644 --- a/src/emqx_bridge.erl +++ b/src/emqx_bridge.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,452 +12,543 @@ %% See the License for the specific language governing permissions and %% limitations under the License. +%% @doc Bridge works in two layers (1) batching layer (2) transport layer +%% The `bridge' batching layer collects local messages in batches and sends over +%% to remote MQTT node/cluster via `connetion' transport layer. +%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be +%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection' +%% has to be `emqx_bridge_mqtt'. +%% +%% ``` +%% +------+ +--------+ +%% | EMQX | | REMOTE | +%% | | | | +%% | (bridge) <==(connection)==> | | +%% | | | | +%% | | | | +%% +------+ +--------+ +%% ''' +%% +%% +%% This module implements 2 kinds of APIs with regards to batching and +%% messaging protocol. (1) A `gen_statem' based local batch collector; +%% (2) APIs for incoming remote batches/messages. +%% +%% Batch collector state diagram +%% +%% [standing_by] --(0) --> [connecting] --(2)--> [connected] +%% | ^ | +%% | | | +%% '--(1)---'--------(3)------' +%% +%% (0): auto or manual start +%% (1): retry timeout +%% (2): successfuly connected to remote node/cluster +%% (3): received {disconnected, conn_ref(), Reason} OR +%% failed to send to remote node/cluster. +%% +%% NOTE: A bridge worker may subscribe to multiple (including wildcard) +%% local topics, and the underlying `emqx_bridge_connect' may subscribe to +%% multiple remote topics, however, worker/connections are not designed +%% to support automatic load-balancing, i.e. in case it can not keep up +%% with the amount of messages comming in, administrator should split and +%% balance topics between worker/connections manually. +%% +%% NOTES: +%% * Local messages are all normalised to QoS-1 when exporting to remote + -module(emqx_bridge). +-behaviour(gen_statem). --behaviour(gen_server). +%% APIs +-export([start_link/2, + import_batch/2, + handle_ack/2, + stop/1]). --include("emqx.hrl"). +%% gen_statem callbacks +-export([terminate/3, code_change/4, init/1, callback_mode/0]). + +%% state functions +-export([standing_by/3, connecting/3, connected/3]). + +%% management APIs +-export([ensure_started/1, ensure_started/2, ensure_stopped/1, ensure_stopped/2, status/1]). +-export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). +-export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]). + +-export_type([config/0, + batch/0, + ack_ref/0]). + +-type id() :: atom() | string() | pid(). +-type qos() :: emqx_mqtt_types:qos(). +-type config() :: map(). +-type batch() :: [emqx_bridge_msg:exp_msg()]. +-type ack_ref() :: term(). +-type topic() :: emqx_topic:topic(). + +-include("logger.hrl"). -include("emqx_mqtt.hrl"). --import(proplists, [get_value/2, get_value/3]). +%% same as default in-flight limit for emqx_client +-define(DEFAULT_BATCH_COUNT, 32). +-define(DEFAULT_BATCH_BYTES, 1 bsl 20). +-define(DEFAULT_SEND_AHEAD, 8). +-define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)). +-define(DEFAULT_SEG_BYTES, (1 bsl 20)). +-define(maybe_send, {next_event, internal, maybe_send}). --export([start_link/2, start_bridge/1, stop_bridge/1, status/1]). +%% @doc Start a bridge worker. Supported configs: +%% start_type: 'manual' (default) or 'auto', when manual, bridge will stay +%% at 'standing_by' state until a manual call to start it. +%% connect_module: The module which implements emqx_bridge_connect behaviour +%% and work as message batch transport layer +%% reconnect_delay_ms: Delay in milli-seconds for the bridge worker to retry +%% in case of transportation failure. +%% max_inflight_batches: Max number of batches allowed to send-ahead before +%% receiving confirmation from remote node/cluster +%% mountpoint: The topic mount point for messages sent to remote node/cluster +%% `undefined', `<<>>' or `""' to disable +%% forwards: Local topics to subscribe. +%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each +%% send call towards emqx_bridge_connect +%% queue.batch_count_limit: Max number of messages to collect in a batch for +%% each send call towards emqx_bridge_connect +%% queue.replayq_dir: Directory where replayq should persist messages +%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file +%% +%% Find more connection specific configs in the callback modules +%% of emqx_bridge_connect behaviour. +start_link(Name, Config) when is_list(Config) -> + start_link(Name, maps:from_list(Config)); +start_link(Name, Config) -> + gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []). --export([show_forwards/1, add_forward/2, del_forward/2]). +%% @doc Manually start bridge worker. State idempotency ensured. +ensure_started(Name) -> + gen_statem:call(name(Name), ensure_started). --export([show_subscriptions/1, add_subscription/3, del_subscription/2]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --record(state, {client_pid :: pid(), - options :: list(), - reconnect_interval :: pos_integer(), - mountpoint :: binary(), - readq :: list(), - writeq :: list(), - replayq :: map(), - ackref :: replayq:ack_ref(), - queue_option :: map(), - forwards :: list(), - subscriptions :: list()}). - --record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, - packet_id, topic, props, payload}). - -start_link(Name, Options) -> - gen_server:start_link({local, name(Name)}, ?MODULE, [Options], []). - -start_bridge(Name) -> - gen_server:call(name(Name), start_bridge). - -stop_bridge(Name) -> - gen_server:call(name(Name), stop_bridge). - --spec(show_forwards(atom()) -> list()). -show_forwards(Name) -> - gen_server:call(name(Name), show_forwards). - --spec(add_forward(atom(), binary()) -> ok | {error, already_exists | validate_fail}). -add_forward(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {add_forward, Topic}) - catch - _Error:_Reason -> - {error, validate_fail} +ensure_started(Name, Config) -> + case start_link(Name, Config) of + {ok, Pid} -> {ok, Pid}; + {error, {already_started,Pid}} -> {ok, Pid} end. --spec(del_forward(atom(), binary()) -> ok | {error, validate_fail}). -del_forward(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {del_forward, Topic}) - catch - _Error:_Reason -> - {error, validate_fail} +%% @doc Manually stop bridge worker. State idempotency ensured. +ensure_stopped(Id) -> + ensure_stopped(Id, 1000). + +ensure_stopped(Id, Timeout) -> + Pid = case id(Id) of + P when is_pid(P) -> P; + N -> whereis(N) + end, + case Pid of + undefined -> + ok; + _ -> + MRef = monitor(process, Pid), + unlink(Pid), + _ = gen_statem:call(id(Id), ensure_stopped, Timeout), + receive + {'DOWN', MRef, _, _, _} -> + ok + after + Timeout -> + exit(Pid, kill) + end end. --spec(show_subscriptions(atom()) -> list()). -show_subscriptions(Name) -> - gen_server:call(name(Name), show_subscriptions). - --spec(add_subscription(atom(), binary(), integer()) -> ok | {error, already_exists | validate_fail}). -add_subscription(Name, Topic, QoS) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {add_subscription, Topic, QoS}) - catch - _Error:_Reason -> - {error, validate_fail} - end. - --spec(del_subscription(atom(), binary()) -> ok | {error, validate_fail}). -del_subscription(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {del_subscription, Topic}) - catch - error:_Reason -> - {error, validate_fail} - end. +stop(Pid) -> gen_statem:stop(Pid). status(Pid) -> - gen_server:call(Pid, status). + gen_statem:call(Pid, status). -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ +%% @doc This function is to be evaluated on message/batch receiver side. +-spec import_batch(batch(), fun(() -> ok)) -> ok. +import_batch(Batch, AckFun) -> + lists:foreach(fun emqx_broker:publish/1, emqx_bridge_msg:to_broker_msgs(Batch)), + AckFun(). -init([Options]) -> - process_flag(trap_exit, true), - case get_value(start_type, Options, manual) of - manual -> ok; - auto -> erlang:send_after(1000, self(), start) - end, - ReconnectInterval = get_value(reconnect_interval, Options, 30000), - Mountpoint = format_mountpoint(get_value(mountpoint, Options)), - QueueOptions = get_value(queue, Options), - {ok, #state{mountpoint = Mountpoint, - queue_option = QueueOptions, - readq = [], - writeq = [], - options = Options, - reconnect_interval = ReconnectInterval}}. - -handle_call(start_bridge, _From, State = #state{client_pid = undefined}) -> - {Msg, NewState} = bridge(start, State), - {reply, #{msg => Msg}, NewState}; - -handle_call(start_bridge, _From, State) -> - {reply, #{msg => <<"bridge already started">>}, State}; - -handle_call(stop_bridge, _From, State = #state{client_pid = undefined}) -> - {reply, #{msg => <<"bridge not started">>}, State}; - -handle_call(stop_bridge, _From, State = #state{client_pid = Pid}) -> - emqx_client:disconnect(Pid), - {reply, #{msg => <<"stop bridge successfully">>}, State}; - -handle_call(status, _From, State = #state{client_pid = undefined}) -> - {reply, #{status => <<"Stopped">>}, State}; -handle_call(status, _From, State = #state{client_pid = _Pid})-> - {reply, #{status => <<"Running">>}, State}; - -handle_call(show_forwards, _From, State = #state{forwards = Forwards}) -> - {reply, Forwards, State}; - -handle_call({add_forward, Topic}, _From, State = #state{forwards = Forwards}) -> - case not lists:member(Topic, Forwards) of - true -> - emqx_broker:subscribe(Topic), - {reply, ok, State#state{forwards = [Topic | Forwards]}}; - false -> - {reply, {error, already_exists}, State} - end; - -handle_call({del_forward, Topic}, _From, State = #state{forwards = Forwards}) -> - case lists:member(Topic, Forwards) of - true -> - emqx_broker:unsubscribe(Topic), - {reply, ok, State#state{forwards = lists:delete(Topic, Forwards)}}; - false -> - {reply, ok, State} - end; - -handle_call(show_subscriptions, _From, State = #state{subscriptions = Subscriptions}) -> - {reply, Subscriptions, State}; - -handle_call({add_subscription, Topic, Qos}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> - case not lists:keymember(Topic, 1, Subscriptions) of - true -> - emqx_client:subscribe(ClientPid, {Topic, Qos}), - {reply, ok, State#state{subscriptions = [{Topic, Qos} | Subscriptions]}}; - false -> - {reply, {error, already_exists}, State} - end; - -handle_call({del_subscription, Topic}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> - case lists:keymember(Topic, 1, Subscriptions) of - true -> - emqx_client:unsubscribe(ClientPid, Topic), - {reply, ok, State#state{subscriptions = lists:keydelete(Topic, 1, Subscriptions)}}; - false -> - {reply, ok, State} - end; - -handle_call(Req, _From, State) -> - emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), - {noreply, State}. - -%%---------------------------------------------------------------- -%% Start or restart bridge -%%---------------------------------------------------------------- -handle_info(start, State) -> - {_Msg, NewState} = bridge(start, State), - {noreply, NewState}; - -handle_info(restart, State) -> - {_Msg, NewState} = bridge(restart, State), - {noreply, NewState}; - -%%---------------------------------------------------------------- -%% pop message from replayq and publish again -%%---------------------------------------------------------------- -handle_info(pop, State = #state{writeq = WriteQ, replayq = ReplayQ, - queue_option = #{batch_size := BatchSize}}) -> - {NewReplayQ, AckRef, NewReadQ} = replayq:pop(ReplayQ, #{count_limit => BatchSize}), - {NewReadQ1, NewWriteQ} = case NewReadQ of - [] -> {WriteQ, []}; - _ -> {NewReadQ, WriteQ} - end, - self() ! replay, - {noreply, State#state{readq = NewReadQ1, writeq = NewWriteQ, replayq = NewReplayQ, ackref = AckRef}}; - -handle_info(dump, State = #state{writeq = WriteQ, replayq = ReplayQ}) -> - NewReplayQueue = replayq:append(ReplayQ, lists:reverse(WriteQ)), - {noreply, State#state{replayq = NewReplayQueue, writeq = []}}; - -%%---------------------------------------------------------------- -%% replay message from replayq -%%---------------------------------------------------------------- -handle_info(replay, State = #state{client_pid = ClientPid, readq = ReadQ}) -> - {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []), - {noreply, State#state{readq = NewReadQ}}; - -%%---------------------------------------------------------------- -%% received local node message -%%---------------------------------------------------------------- -handle_info({dispatch, _, #message{topic = Topic, qos = QoS, payload = Payload, flags = #{retain := Retain}}}, - State = #state{client_pid = undefined, - mountpoint = Mountpoint}) - when QoS =< 1 -> - Msg = #mqtt_msg{qos = 1, - retain = Retain, - topic = mountpoint(Mountpoint, Topic), - payload = Payload}, - {noreply, en_writeq({undefined, Msg}, State)}; -handle_info({dispatch, _, #message{topic = Topic, qos = QoS ,payload = Payload, flags = #{retain := Retain}}}, - State = #state{client_pid = Pid, - mountpoint = Mountpoint}) - when QoS =< 1 -> - Msg = #mqtt_msg{qos = 1, - retain = Retain, - topic = mountpoint(Mountpoint, Topic), - payload = Payload}, - case emqx_client:publish(Pid, Msg) of - {ok, PktId} -> - {noreply, en_writeq({PktId, Msg}, State)}; - {error, {PktId, Reason}} -> - emqx_logger:error("[Bridge] Publish fail:~p", [Reason]), - {noreply, en_writeq({PktId, Msg}, State)} - end; - -%%---------------------------------------------------------------- -%% received remote node message -%%---------------------------------------------------------------- -handle_info({publish, #{qos := QoS, dup := Dup, retain := Retain, topic := Topic, - properties := Props, payload := Payload}}, State) -> - NewMsg0 = emqx_message:make(bridge, QoS, Topic, Payload), - NewMsg1 = emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, NewMsg0)), - emqx_broker:publish(NewMsg1), - {noreply, State}; - -%%---------------------------------------------------------------- -%% received remote puback message -%%---------------------------------------------------------------- -handle_info({puback, #{packet_id := PktId}}, State) -> - {noreply, delete(PktId, State)}; - -handle_info({'EXIT', Pid, normal}, State = #state{client_pid = Pid}) -> - emqx_logger:warning("[Bridge] stop ~p", [normal]), - self() ! dump, - {noreply, State#state{client_pid = undefined}}; - -handle_info({'EXIT', Pid, Reason}, State = #state{client_pid = Pid, - reconnect_interval = ReconnectInterval}) -> - emqx_logger:error("[Bridge] stop ~p", [Reason]), - self() ! dump, - erlang:send_after(ReconnectInterval, self(), restart), - {noreply, State#state{client_pid = undefined}}; - -handle_info(Info, State) -> - emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{}) -> +%% @doc This function is to be evaluated on message/batch exporter side +%% when message/batch is accepted by remote node. +-spec handle_ack(pid(), ack_ref()) -> ok. +handle_ack(Pid, Ref) when node() =:= node(Pid) -> + Pid ! {batch_ack, Ref}, ok. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +%% @doc Return all forwards (local subscriptions). +-spec get_forwards(id()) -> [topic()]. +get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)). -subscribe_remote_topics(ClientPid, Subscriptions) -> - [begin emqx_client:subscribe(ClientPid, {bin(Topic), Qos}), {bin(Topic), Qos} end - || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})]. +%% @doc Return all subscriptions (subscription over mqtt connection to remote broker). +-spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. +get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). -subscribe_local_topics(Options) -> - Topics = get_value(forwards, Options, []), - Subid = get_value(client_id, Options, <<"bridge">>), - [begin emqx_broker:subscribe(bin(Topic), #{qos => 1, subid => Subid}), bin(Topic) end - || Topic <- Topics, emqx_topic:validate({filter, bin(Topic)})]. +%% @doc Add a new forward (local topic subscription). +-spec ensure_forward_present(id(), topic()) -> ok. +ensure_forward_present(Id, Topic) -> + gen_statem:call(id(Id), {ensure_present, forwards, topic(Topic)}). -proto_ver(mqttv3) -> v3; -proto_ver(mqttv4) -> v4; -proto_ver(mqttv5) -> v5. -address(Address) -> - case string:tokens(Address, ":") of - [Host] -> {Host, 1883}; - [Host, Port] -> {Host, list_to_integer(Port)} +%% @doc Ensure a forward topic is deleted. +-spec ensure_forward_absent(id(), topic()) -> ok. +ensure_forward_absent(Id, Topic) -> + gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}). + +%% @doc Ensure subscribed to remote topic. +%% NOTE: only applicable when connection module is emqx_bridge_mqtt +%% return `{error, no_remote_subscription_support}' otherwise. +-spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. +ensure_subscription_present(Id, Topic, QoS) -> + gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}). + +%% @doc Ensure unsubscribed from remote topic. +%% NOTE: only applicable when connection module is emqx_bridge_mqtt +-spec ensure_subscription_absent(id(), topic()) -> ok. +ensure_subscription_absent(Id, Topic) -> + gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}). + +callback_mode() -> [state_functions, state_enter]. + +%% @doc Config should be a map(). +init(Config) -> + erlang:process_flag(trap_exit, true), + Get = fun(K, D) -> maps:get(K, Config, D) end, + QCfg = maps:get(queue, Config, #{}), + GetQ = fun(K, D) -> maps:get(K, QCfg, D) end, + Dir = GetQ(replayq_dir, undefined), + QueueConfig = + case Dir =:= undefined orelse Dir =:= "" of + true -> #{mem_only => true}; + false -> #{dir => Dir, + seg_bytes => GetQ(replayq_seg_bytes, ?DEFAULT_SEG_BYTES) + } + end, + Queue = replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1, + marshaller => fun msg_marshaller/1}), + Topics = lists:sort([iolist_to_binary(T) || T <- Get(forwards, [])]), + Subs = lists:keysort(1, lists:map(fun({T0, QoS}) -> + T = iolist_to_binary(T0), + true = emqx_topic:validate({filter, T}), + {T, QoS} + end, Get(subscriptions, []))), + ConnectModule = maps:get(connect_module, Config), + ConnectConfig = maps:without([connect_module, + queue, + reconnect_delay_ms, + max_inflight_batches, + mountpoint, + forwards + ], Config#{subscriptions => Subs}), + ConnectFun = fun(SubsX) -> emqx_bridge_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end, + {ok, standing_by, + #{connect_module => ConnectModule, + connect_fun => ConnectFun, + start_type => Get(start_type, manual), + reconnect_delay_ms => maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS), + batch_bytes_limit => GetQ(batch_bytes_limit, ?DEFAULT_BATCH_BYTES), + batch_count_limit => GetQ(batch_count_limit, ?DEFAULT_BATCH_COUNT), + max_inflight_batches => Get(max_inflight_batches, ?DEFAULT_SEND_AHEAD), + mountpoint => format_mountpoint(Get(mountpoint, undefined)), + forwards => Topics, + subscriptions => Subs, + replayq => Queue, + inflight => [] + }}. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +terminate(_Reason, _StateName, #{replayq := Q} = State) -> + _ = disconnect(State), + _ = replayq:close(Q), + ok. + +%% @doc Standing by for manual start. +standing_by(enter, _, #{start_type := auto}) -> + Action = {state_timeout, 0, do_connect}, + {keep_state_and_data, Action}; +standing_by(enter, _, #{start_type := manual}) -> + keep_state_and_data; +standing_by({call, From}, ensure_started, State) -> + {next_state, connecting, State, + [{reply, From, ok}]}; +standing_by(state_timeout, do_connect, State) -> + {next_state, connecting, State}; +standing_by({call, From}, _Call, _State) -> + {keep_state_and_data, [{reply, From, {error,standing_by}}]}; +standing_by(info, Info, State) -> + ?LOG(info, "Bridge ~p discarded info event at state standing_by:\n~p", [name(), Info]), + {keep_state_and_data, State}; +standing_by(Type, Content, State) -> + common(standing_by, Type, Content, State). + +%% @doc Connecting state is a state with timeout. +%% After each timeout, it re-enters this state and start a retry until +%% successfuly connected to remote node/cluster. +connecting(enter, connected, #{reconnect_delay_ms := Timeout}) -> + Action = {state_timeout, Timeout, reconnect}, + {keep_state_and_data, Action}; +connecting(enter, _, #{reconnect_delay_ms := Timeout, + connect_fun := ConnectFun, + subscriptions := Subs, + forwards := Forwards + } = State) -> + ok = subscribe_local_topics(Forwards), + case ConnectFun(Subs) of + {ok, ConnRef, Conn} -> + ?LOG(info, "Bridge ~p connected", [name()]), + Action = {state_timeout, 0, connected}, + {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; + error -> + Action = {state_timeout, Timeout, reconnect}, + {keep_state_and_data, Action} + end; +connecting(state_timeout, connected, State) -> + {next_state, connected, State}; +connecting(state_timeout, reconnect, _State) -> + repeat_state_and_data; +connecting(info, {batch_ack, Ref}, State) -> + case do_ack(State, Ref) of + {ok, NewState} -> + {keep_state, NewState}; + _ -> + keep_state_and_data + end; +connecting(internal, maybe_send, _State) -> + keep_state_and_data; +connecting(info, {disconnected, _Ref, _Reason}, _State) -> + keep_state_and_data; +connecting(Type, Content, State) -> + common(connecting, Type, Content, State). + +%% @doc Send batches to remote node/cluster when in 'connected' state. +connected(enter, _OldState, #{inflight := Inflight} = State) -> + case retry_inflight(State#{inflight := []}, Inflight) of + {ok, NewState} -> + Action = {state_timeout, 0, success}, + {keep_state, NewState, Action}; + {error, NewState} -> + Action = {state_timeout, 0, failure}, + {keep_state, disconnect(NewState), Action} + end; +connected(state_timeout, failure, State) -> + {next_state, connecting, State}; +connected(state_timeout, success, State) -> + {keep_state, State, ?maybe_send}; +connected(internal, maybe_send, State) -> + case pop_and_send(State) of + {ok, NewState} -> + {keep_state, NewState}; + {error, NewState} -> + {next_state, connecting, disconnect(NewState)} + end; +connected(info, {disconnected, ConnRef, Reason}, + #{conn_ref := ConnRefCurrent, connection := Conn} = State) -> + case ConnRefCurrent =:= ConnRef of + true -> + ?LOG(info, "Bridge ~p diconnected~nreason=~p", [name(), Conn, Reason]), + {next_state, connecting, + State#{conn_ref := undefined, connection := undefined}}; + false -> + keep_state_and_data + end; +connected(info, {batch_ack, Ref}, State) -> + case do_ack(State, Ref) of + stale -> + keep_state_and_data; + bad_order -> + %% try re-connect then re-send + ?LOG(error, "Bad order ack received by bridge ~p", [name()]), + {next_state, connecting, disconnect(State)}; + {ok, NewState} -> + {keep_state, NewState, ?maybe_send} + end; +connected(Type, Content, State) -> + common(connected, Type, Content, State). + +%% Common handlers +common(StateName, {call, From}, status, _State) -> + {keep_state_and_data, [{reply, From, StateName}]}; +common(_StateName, {call, From}, ensure_started, _State) -> + {keep_state_and_data, [{reply, From, ok}]}; +common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> + {keep_state_and_data, [{reply, From, Forwards}]}; +common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> + {keep_state_and_data, [{reply, From, Subs}]}; +common(_StateName, {call, From}, {ensure_present, What, Topic}, State) -> + {Result, NewState} = ensure_present(What, Topic, State), + {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> + {Result, NewState} = ensure_absent(What, Topic, State), + {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, ensure_stopped, _State) -> + {stop_and_reply, {shutdown, manual}, + [{reply, From, ok}]}; +common(_StateName, info, {dispatch, _, Msg}, + #{replayq := Q} = State) -> + NewQ = replayq:append(Q, collect([Msg])), + {keep_state, State#{replayq => NewQ}, ?maybe_send}; +common(StateName, Type, Content, State) -> + ?LOG(info, "Bridge ~p discarded ~p type event at state ~p:\n~p", + [name(), Type, StateName, Content]), + {keep_state, State}. + +ensure_present(Key, Topic, State) -> + Topics = maps:get(Key, State), + case is_topic_present(Topic, Topics) of + true -> + {ok, State}; + false -> + R = do_ensure_present(Key, Topic, State), + {R, State#{Key := lists:usort([Topic | Topics])}} end. -options(Options) -> - options(Options, []). -options([], Acc) -> - Acc; -options([{username, Username}| Options], Acc) -> - options(Options, [{username, Username}|Acc]); -options([{proto_ver, ProtoVer}| Options], Acc) -> - options(Options, [{proto_ver, proto_ver(ProtoVer)}|Acc]); -options([{password, Password}| Options], Acc) -> - options(Options, [{password, Password}|Acc]); -options([{keepalive, Keepalive}| Options], Acc) -> - options(Options, [{keepalive, Keepalive}|Acc]); -options([{client_id, ClientId}| Options], Acc) -> - options(Options, [{client_id, ClientId}|Acc]); -options([{clean_start, CleanStart}| Options], Acc) -> - options(Options, [{clean_start, CleanStart}|Acc]); -options([{address, Address}| Options], Acc) -> - {Host, Port} = address(Address), - options(Options, [{host, Host}, {port, Port}|Acc]); -options([{ssl, Ssl}| Options], Acc) -> - options(Options, [{ssl, Ssl}|Acc]); -options([{ssl_opts, SslOpts}| Options], Acc) -> - options(Options, [{ssl_opts, SslOpts}|Acc]); -options([_Option | Options], Acc) -> - options(Options, Acc). -name(Id) -> - list_to_atom(lists:concat([?MODULE, "_", Id])). +ensure_absent(Key, Topic, State) -> + Topics = maps:get(Key, State), + case is_topic_present(Topic, Topics) of + true -> + R = do_ensure_absent(Key, Topic, State), + {R, State#{Key := ensure_topic_absent(Topic, Topics)}}; + false -> + {ok, State} + end. -bin(L) -> iolist_to_binary(L). +ensure_topic_absent(_Topic, []) -> []; +ensure_topic_absent(Topic, [{_, _} | _] = L) -> lists:keydelete(Topic, 1, L); +ensure_topic_absent(Topic, L) -> lists:delete(Topic, L). -mountpoint(undefined, Topic) -> - Topic; -mountpoint(Prefix, Topic) -> - <>. +is_topic_present({Topic, _QoS}, Topics) -> + is_topic_present(Topic, Topics); +is_topic_present(Topic, Topics) -> + lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics). + +do_ensure_present(forwards, Topic, _) -> + ok = subscribe_local_topic(Topic); +do_ensure_present(subscriptions, {Topic, QoS}, + #{connect_module := ConnectModule, connection := Conn}) -> + case erlang:function_exported(ConnectModule, ensure_subscribed, 3) of + true -> + _ = ConnectModule:ensure_subscribed(Conn, Topic, QoS), + ok; + false -> + {error, no_remote_subscription_support} + end. + +do_ensure_absent(forwards, Topic, _) -> + ok = emqx_broker:unsubscribe(Topic); +do_ensure_absent(subscriptions, Topic, #{connect_module := ConnectModule, + connection := Conn}) -> + case erlang:function_exported(ConnectModule, ensure_unsubscribed, 2) of + true -> ConnectModule:ensure_unsubscribed(Conn, Topic); + false -> {error, no_remote_subscription_support} + end. + +collect(Acc) -> + receive + {dispatch, _, Msg} -> + collect([Msg | Acc]) + after + 0 -> + lists:reverse(Acc) + end. + +%% Retry all inflight (previously sent but not acked) batches. +retry_inflight(State, []) -> {ok, State}; +retry_inflight(#{inflight := Inflight} = State, + [#{q_ack_ref := QAckRef, batch := Batch} | T] = Remain) -> + case do_send(State, QAckRef, Batch) of + {ok, NewState} -> + retry_inflight(NewState, T); + {error, Reason} -> + ?LOG(error, "Inflight retry failed\n~p", [Reason]), + {error, State#{inflight := Inflight ++ Remain}} + end. + +pop_and_send(#{inflight := Inflight, + max_inflight_batches := Max + } = State) when length(Inflight) >= Max -> + {ok, State}; +pop_and_send(#{replayq := Q, + batch_count_limit := CountLimit, + batch_bytes_limit := BytesLimit + } = State) -> + case replayq:is_empty(Q) of + true -> + {ok, State}; + false -> + Opts = #{count_limit => CountLimit, bytes_limit => BytesLimit}, + {Q1, QAckRef, Batch} = replayq:pop(Q, Opts), + do_send(State#{replayq := Q1}, QAckRef, Batch) + end. + +%% Assert non-empty batch because we have a is_empty check earlier. +do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) -> + case maybe_send(State, Batch) of + {ok, Ref} -> + %% this is a list of inflight BATCHes, not expecting it to be too long + NewInflight = Inflight ++ [#{q_ack_ref => QAckRef, + send_ack_ref => Ref, + batch => Batch}], + {ok, State#{inflight := NewInflight}}; + {error, Reason} -> + ?LOG(info, "Batch produce failed\n~p", [Reason]), + {error, State} + end. + +do_ack(State = #{inflight := [#{send_ack_ref := Refx, q_ack_ref := QAckRef} | Rest], + replayq := Q}, Ref) when Refx =:= Ref -> + ok = replayq:ack(Q, QAckRef), + {ok, State#{inflight := Rest}}; +do_ack(#{inflight := Inflight}, Ref) -> + case lists:any(fun(#{send_ack_ref := Ref0}) -> Ref0 =:= Ref end, Inflight) of + true -> bad_order; + false -> stale + end. + +subscribe_local_topics(Topics) -> lists:foreach(fun subscribe_local_topic/1, Topics). + +subscribe_local_topic(Topic0) -> + Topic = topic(Topic0), + try + emqx_topic:validate({filter, Topic}) + catch + error : Reason -> + erlang:error({bad_topic, Topic, Reason}) + end, + ok = emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}). + +topic(T) -> iolist_to_binary(T). + +disconnect(#{connection := Conn, + conn_ref := ConnRef, + connect_module := Module + } = State) when Conn =/= undefined -> + ok = Module:stop(ConnRef, Conn), + State#{conn_ref => undefined, + connection => undefined}; +disconnect(State) -> State. + +%% Called only when replayq needs to dump it to disk. +msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin); +msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg). + +%% Return {ok, SendAckRef} or {error, Reason} +maybe_send(#{connect_module := Module, + connection := Connection, + mountpoint := Mountpoint + }, Batch) -> + Module:send(Connection, [emqx_bridge_msg:to_export(Module, Mountpoint, M) || M <- Batch]). format_mountpoint(undefined) -> undefined; format_mountpoint(Prefix) -> - binary:replace(bin(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). + binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). -en_writeq(Msg, State = #state{replayq = ReplayQ, - queue_option = #{mem_cache := false}}) -> - NewReplayQ = replayq:append(ReplayQ, [Msg]), - State#state{replayq = NewReplayQ}; -en_writeq(Msg, State = #state{writeq = WriteQ, - queue_option = #{batch_size := BatchSize, - mem_cache := true}}) - when length(WriteQ) < BatchSize-> - State#state{writeq = [Msg | WriteQ]} ; -en_writeq(Msg, State = #state{writeq = WriteQ, replayq = ReplayQ, - queue_option = #{mem_cache := true}}) -> - NewReplayQ =replayq:append(ReplayQ, lists:reverse(WriteQ)), - State#state{writeq = [Msg], replayq = NewReplayQ}. +name() -> {_, Name} = process_info(self(), registered_name), Name. -publish_readq_msg(_ClientPid, [], NewReadQ) -> - {ok, NewReadQ}; -publish_readq_msg(ClientPid, [{_PktId, Msg} | ReadQ], NewReadQ) -> - {ok, PktId} = emqx_client:publish(ClientPid, Msg), - publish_readq_msg(ClientPid, ReadQ, [{PktId, Msg} | NewReadQ]). +name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). -delete(PktId, State = #state{ replayq = ReplayQ, - readq = [], - queue_option = #{ mem_cache := false}}) -> - {NewReplayQ, NewAckRef, Msgs} = replayq:pop(ReplayQ, #{count_limit => 1}), - logger:debug("[Msg] PacketId ~p, Msg: ~p", [PktId, Msgs]), - ok = replayq:ack(NewReplayQ, NewAckRef), - case Msgs of - [{PktId, _Msg}] -> - self() ! pop, - State#state{ replayq = NewReplayQ, ackref = NewAckRef }; - [{_PktId, _Msg}] -> - NewReplayQ1 = replayq:append(NewReplayQ, Msgs), - self() ! pop, - State#state{ replayq = NewReplayQ1, ackref = NewAckRef }; - _Empty -> - State#state{ replayq = NewReplayQ, ackref = NewAckRef} - end; -delete(_PktId, State = #state{readq = [], writeq = [], replayq = ReplayQ, ackref = AckRef}) -> - ok = replayq:ack(ReplayQ, AckRef), - self() ! pop, - State; - -delete(PktId, State = #state{readq = [], writeq = WriteQ}) -> - State#state{writeq = lists:keydelete(PktId, 1, WriteQ)}; - -delete(PktId, State = #state{readq = ReadQ, replayq = ReplayQ, ackref = AckRef}) -> - NewReadQ = lists:keydelete(PktId, 1, ReadQ), - case NewReadQ of - [] -> - ok = replayq:ack(ReplayQ, AckRef), - self() ! pop; - _NewReadQ -> - ok - end, - State#state{ readq = NewReadQ }. - -bridge(Action, State = #state{options = Options, - replayq = ReplayQ, - queue_option - = QueueOption - = #{batch_size := BatchSize}}) - when BatchSize > 0 -> - case emqx_client:start_link([{owner, self()} | options(Options)]) of - {ok, ClientPid} -> - case emqx_client:connect(ClientPid) of - {ok, _} -> - emqx_logger:info("[Bridge] connected to remote successfully"), - Subs = subscribe_remote_topics(ClientPid, get_value(subscriptions, Options, [])), - Forwards = subscribe_local_topics(Options), - {NewReplayQ, AckRef, ReadQ} = open_replayq(ReplayQ, QueueOption), - {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []), - {<<"start bridge successfully">>, - State#state{client_pid = ClientPid, - subscriptions = Subs, - readq = NewReadQ, - replayq = NewReplayQ, - ackref = AckRef, - forwards = Forwards}}; - {error, Reason} -> - emqx_logger:error("[Bridge] connect to remote failed! error: ~p", [Reason]), - {<<"connect to remote failed">>, - State#state{client_pid = ClientPid}} - end; - {error, Reason} -> - emqx_logger:error("[Bridge] ~p failed! error: ~p", [Action, Reason]), - {<<"start bridge failed">>, State} - end; -bridge(Action, State) -> - emqx_logger:error("[Bridge] ~p failed! error: batch_size should greater than zero", [Action]), - {<<"Open Replayq failed">>, State}. - -open_replayq(undefined, #{batch_size := BatchSize, - replayq_dir := ReplayqDir, - replayq_seg_bytes := ReplayqSegBytes}) -> - ReplayQ = replayq:open(#{dir => ReplayqDir, - seg_bytes => ReplayqSegBytes, - sizer => fun(Term) -> - size(term_to_binary(Term)) - end, - marshaller => fun({PktId, Msg}) -> - term_to_binary({PktId, Msg}); - (Bin) -> - binary_to_term(Bin) - end}), - replayq:pop(ReplayQ, #{count_limit => BatchSize}); -open_replayq(ReplayQ, #{batch_size := BatchSize}) -> - replayq:pop(ReplayQ, #{count_limit => BatchSize}). +id(Pid) when is_pid(Pid) -> Pid; +id(Name) -> name(Name). diff --git a/src/emqx_bridge_connect.erl b/src/emqx_bridge_connect.erl new file mode 100644 index 000000000..b2781cc2c --- /dev/null +++ b/src/emqx_bridge_connect.erl @@ -0,0 +1,71 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_connect). + +-export([start/2]). + +-export_type([config/0, connection/0]). + +-optional_callbacks([ensure_subscribed/3, ensure_unsubscribed/2]). + +%% map fields depend on implementation +-type config() :: map(). +-type connection() :: term(). +-type conn_ref() :: term(). +-type batch() :: emqx_protal:batch(). +-type ack_ref() :: emqx_bridge:ack_ref(). +-type topic() :: emqx_topic:topic(). +-type qos() :: emqx_mqtt_types:qos(). + +-include("logger.hrl"). + +%% establish the connection to remote node/cluster +%% protal worker (the caller process) should be expecting +%% a message {disconnected, conn_ref()} when disconnected. +-callback start(config()) -> {ok, conn_ref(), connection()} | {error, any()}. + +%% send to remote node/cluster +%% bridge worker (the caller process) should be expecting +%% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster +-callback send(connection(), batch()) -> {ok, ack_ref()} | {error, any()}. + +%% called when owner is shutting down. +-callback stop(conn_ref(), connection()) -> ok. + +-callback ensure_subscribed(connection(), topic(), qos()) -> ok. + +-callback ensure_unsubscribed(connection(), topic()) -> ok. + +start(Module, Config) -> + case Module:start(Config) of + {ok, Ref, Conn} -> + {ok, Ref, Conn}; + {error, Reason} -> + Config1 = obfuscate(Config), + ?LOG(error, "Failed to connect with module=~p\n" + "config=~p\nreason:~p", [Module, Config1, Reason]), + error + end. + +obfuscate(Map) -> + maps:fold(fun(K, V, Acc) -> + case is_sensitive(K) of + true -> [{K, '***'} | Acc]; + false -> [{K, V} | Acc] + end + end, [], Map). + +is_sensitive(password) -> true; +is_sensitive(_) -> false. diff --git a/src/emqx_bridge_mqtt.erl b/src/emqx_bridge_mqtt.erl new file mode 100644 index 000000000..486f3206a --- /dev/null +++ b/src/emqx_bridge_mqtt.erl @@ -0,0 +1,185 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol + +-module(emqx_bridge_mqtt). +-behaviour(emqx_bridge_connect). + +%% behaviour callbacks +-export([start/1, + send/2, + stop/2 + ]). + +%% optional behaviour callbacks +-export([ensure_subscribed/3, + ensure_unsubscribed/2 + ]). + +-include("emqx_mqtt.hrl"). + +-define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}). + +%% Messages towards ack collector process +-define(RANGE(Min, Max), {Min, Max}). +-define(REF_IDS(Ref, Ids), {Ref, Ids}). +-define(SENT(RefIds), {sent, RefIds}). +-define(ACKED(AnyPktId), {acked, AnyPktId}). +-define(STOP(Ref), {stop, Ref}). + +start(Config = #{address := Address}) -> + Ref = make_ref(), + Parent = self(), + AckCollector = spawn_link(fun() -> ack_collector(Parent, Ref) end), + Handlers = make_hdlr(Parent, AckCollector, Ref), + {Host, Port} = case string:tokens(Address, ":") of + [H] -> {H, 1883}; + [H, P] -> {H, list_to_integer(P)} + end, + ClientConfig = Config#{msg_handler => Handlers, + owner => AckCollector, + host => Host, + port => Port}, + case emqx_client:start_link(ClientConfig) of + {ok, Pid} -> + case emqx_client:connect(Pid) of + {ok, _} -> + try + subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])), + {ok, Ref, #{ack_collector => AckCollector, + client_pid => Pid}} + catch + throw : Reason -> + ok = stop(AckCollector, Pid), + {error, Reason} + end; + {error, Reason} -> + ok = stop(Ref, #{ack_collector => AckCollector, client_pid => Pid}), + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +stop(Ref, #{ack_collector := AckCollector, client_pid := Pid}) -> + safe_stop(Pid, fun() -> emqx_client:stop(Pid) end, 1000), + safe_stop(AckCollector, fun() -> AckCollector ! ?STOP(Ref) end, 1000), + ok. + +ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) -> + emqx_client:subscribe(Pid, Topic, QoS); +ensure_subscribed(_Conn, _Topic, _QoS) -> + %% return ok for now, next re-connect should should call start with new topic added to config + ok. + +ensure_unsubscribed(#{client_pid := Pid}, Topic) when is_pid(Pid) -> + emqx_client:unsubscribe(Pid, Topic); +ensure_unsubscribed(_, _) -> + %% return ok for now, next re-connect should should call start with this topic deleted from config + ok. + +safe_stop(Pid, StopF, Timeout) -> + MRef = monitor(process, Pid), + unlink(Pid), + try + StopF() + catch + _ : _ -> + ok + end, + receive + {'DOWN', MRef, _, _, _} -> + ok + after + Timeout -> + exit(Pid, kill) + end. + +send(Conn, Batch) -> + send(Conn, Batch, []). + +send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, [Msg | Rest], Acc) -> + case emqx_client:publish(ClientPid, Msg) of + {ok, PktId} when Rest =:= [] -> + %% last one sent + Ref = make_ref(), + AckCollector ! ?SENT(?REF_IDS(Ref, lists:reverse([PktId | Acc]))), + {ok, Ref}; + {ok, PktId} -> + send(Conn, Rest, [PktId | Acc]); + {error, Reason} -> + %% NOTE: There is no partial sucess of a batch and recover from the middle + %% only to retry all messages in one batch + {error, Reason} + end. + +ack_collector(Parent, ConnRef) -> + ack_collector(Parent, ConnRef, queue:new(), []). + +ack_collector(Parent, ConnRef, Acked, Sent) -> + {NewAcked, NewSent} = + receive + ?STOP(ConnRef) -> + exit(normal); + ?ACKED(PktId) -> + match_acks(Parent, queue:in(PktId, Acked), Sent); + ?SENT(RefIds) -> + %% this message only happens per-batch, hence ++ is ok + match_acks(Parent, Acked, Sent ++ [RefIds]) + after + 200 -> + {Acked, Sent} + end, + ack_collector(Parent, ConnRef, NewAcked, NewSent). + +match_acks(_Parent, Acked, []) -> {Acked, []}; +match_acks(Parent, Acked, Sent) -> + match_acks_1(Parent, queue:out(Acked), Sent). + +match_acks_1(_Parent, {empty, Empty}, Sent) -> {Empty, Sent}; +match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId]) | Sent]) -> + %% batch finished + ok = emqx_bridge:handle_ack(Parent, Ref), + match_acks(Parent, Acked, Sent); +match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId | RestIds]) | Sent]) -> + %% one message finished, but not the whole batch + match_acks(Parent, Acked, [?REF_IDS(Ref, RestIds) | Sent]). + + +%% When puback for QoS-1 message is received from remote MQTT broker +%% NOTE: no support for QoS-2 +handle_puback(AckCollector, #{packet_id := PktId, reason_code := RC}) -> + RC =:= ?RC_SUCCESS orelse error({puback_error_code, RC}), + AckCollector ! ?ACKED(PktId), + ok. + +%% Message published from remote broker. Import to local broker. +import_msg(Msg) -> + %% auto-ack should be enabled in emqx_client, hence dummy ack-fun. + emqx_bridge:import_batch([Msg], _AckFun = fun() -> ok end). + +make_hdlr(Parent, AckCollector, Ref) -> + #{puback => fun(Ack) -> handle_puback(AckCollector, Ack) end, + publish => fun(Msg) -> import_msg(Msg) end, + disconnected => fun(Reason) -> Parent ! {disconnected, Ref, Reason}, ok end + }. + +subscribe_remote_topics(ClientPid, Subscriptions) -> + lists:foreach(fun({Topic, Qos}) -> + case emqx_client:subscribe(ClientPid, Topic, Qos) of + {ok, _, _} -> ok; + Error -> throw(Error) + end + end, Subscriptions). diff --git a/src/emqx_bridge_msg.erl b/src/emqx_bridge_msg.erl new file mode 100644 index 000000000..6633027f9 --- /dev/null +++ b/src/emqx_bridge_msg.erl @@ -0,0 +1,84 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_msg). + +-export([ to_binary/1 + , from_binary/1 + , to_export/3 + , to_broker_msgs/1 + , estimate_size/1 + ]). + +-export_type([msg/0]). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include("emqx_client.hrl"). + +-type msg() :: emqx_types:message(). +-type exp_msg() :: emqx_types:message() | #mqtt_msg{}. + +%% @doc Make export format: +%% 1. Mount topic to a prefix +%% 2. Fix QoS to 1 +%% @end +%% Shame that we have to know the callback module here +%% would be great if we can get rid of #mqtt_msg{} record +%% and use #message{} in all places. +-spec to_export(emqx_bridge_rpc | emqx_bridge_mqtt, + undefined | binary(), msg()) -> exp_msg(). +to_export(emqx_bridge_mqtt, Mountpoint, + #message{topic = Topic, + payload = Payload, + flags = Flags + }) -> + Retain = maps:get(retain, Flags, false), + #mqtt_msg{qos = ?QOS_1, + retain = Retain, + topic = topic(Mountpoint, Topic), + payload = Payload}; +to_export(_Module, Mountpoint, + #message{topic = Topic} = Msg) -> + Msg#message{topic = topic(Mountpoint, Topic), qos = 1}. + +%% @doc Make `binary()' in order to make iodata to be persisted on disk. +-spec to_binary(msg()) -> binary(). +to_binary(Msg) -> term_to_binary(Msg). + +%% @doc Unmarshal binary into `msg()'. +-spec from_binary(binary()) -> msg(). +from_binary(Bin) -> binary_to_term(Bin). + +%% @doc Estimate the size of a message. +%% Count only the topic length + payload size +-spec estimate_size(msg()) -> integer(). +estimate_size(#message{topic = Topic, payload = Payload}) -> + size(Topic) + size(Payload). + +%% @doc By message/batch receiver, transform received batch into +%% messages to dispatch to local brokers. +to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch). + +to_broker_msg(#message{} = Msg) -> + %% internal format from another EMQX node via rpc + Msg; +to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, + properties := Props, payload := Payload}) -> + %% published from remote node over a MQTT connection + emqx_message:set_headers(Props, + emqx_message:set_flags(#{dup => Dup, retain => Retain}, + emqx_message:make(bridge, QoS, Topic, Payload))). + +topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/src/emqx_bridge_rpc.erl b/src/emqx_bridge_rpc.erl new file mode 100644 index 000000000..b818d65da --- /dev/null +++ b/src/emqx_bridge_rpc.erl @@ -0,0 +1,105 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc This module implements EMQX Bridge transport layer based on gen_rpc. + +-module(emqx_bridge_rpc). +-behaviour(emqx_bridge_connect). + +%% behaviour callbacks +-export([start/1, + send/2, + stop/2 + ]). + +%% Internal exports +-export([ handle_send/2 + , handle_ack/2 + , heartbeat/2 + ]). + +-type ack_ref() :: emqx_bridge:ack_ref(). +-type batch() :: emqx_bridge:batch(). + +-define(HEARTBEAT_INTERVAL, timer:seconds(1)). + +-define(RPC, gen_rpc). + +start(#{address := Remote}) -> + case poke(Remote) of + ok -> + Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), Remote]), + {ok, Pid, Remote}; + Error -> + Error + end. + +stop(Pid, _Remote) when is_pid(Pid) -> + Ref = erlang:monitor(process, Pid), + unlink(Pid), + Pid ! stop, + receive + {'DOWN', Ref, process, Pid, _Reason} -> + ok + after + 1000 -> + exit(Pid, kill) + end, + ok. + +%% @doc Callback for `emqx_bridge_connect' behaviour +-spec send(node(), batch()) -> {ok, ack_ref()} | {error, any()}. +send(Remote, Batch) -> + Sender = self(), + case ?RPC:call(Remote, ?MODULE, handle_send, [Sender, Batch]) of + {ok, Ref} -> {ok, Ref}; + {badrpc, Reason} -> {error, Reason} + end. + +%% @doc Handle send on receiver side. +-spec handle_send(pid(), batch()) -> {ok, ack_ref()} | {error, any()}. +handle_send(SenderPid, Batch) -> + SenderNode = node(SenderPid), + Ref = make_ref(), + AckFun = fun() -> ?RPC:cast(SenderNode, ?MODULE, handle_ack, [SenderPid, Ref]), ok end, + case emqx_bridge:import_batch(Batch, AckFun) of + ok -> {ok, Ref}; + Error -> Error + end. + +%% @doc Handle batch ack in sender node. +handle_ack(SenderPid, Ref) -> + ok = emqx_bridge:handle_ack(SenderPid, Ref). + +%% @hidden Heartbeat loop +heartbeat(Parent, RemoteNode) -> + Interval = ?HEARTBEAT_INTERVAL, + receive + stop -> exit(normal) + after + Interval -> + case poke(RemoteNode) of + ok -> + ?MODULE:heartbeat(Parent, RemoteNode); + {error, Reason} -> + Parent ! {disconnected, self(), Reason}, + exit(normal) + end + end. + +poke(Node) -> + case ?RPC:call(Node, erlang, node, []) of + Node -> ok; + {badrpc, Reason} -> {error, Reason} + end. diff --git a/src/emqx_bridge_sup.erl b/src/emqx_bridge_sup.erl index baa857074..bcacb411c 100644 --- a/src/emqx_bridge_sup.erl +++ b/src/emqx_bridge_sup.erl @@ -13,33 +13,50 @@ %% limitations under the License. -module(emqx_bridge_sup). - -behavior(supervisor). --include("emqx.hrl"). +-include("logger.hrl"). --export([start_link/0, bridges/0]). - -%% Supervisor callbacks +-export([start_link/0, start_link/1, bridges/0]). +-export([create_bridge/2, drop_bridge/1]). -export([init/1]). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). +-define(SUP, ?MODULE). +-define(WORKER_SUP, emqx_bridge_worker_sup). + +start_link() -> start_link(?SUP). + +start_link(Name) -> + supervisor:start_link({local, Name}, ?MODULE, Name). + +init(?SUP) -> + BridgesConf = emqx_config:get_env(bridges, []), + BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), + SupFlag = #{strategy => one_for_one, + intensity => 100, + period => 10}, + {ok, {SupFlag, BridgeSpec}}. + +bridge_spec({Name, Config}) -> + #{id => Name, + start => {emqx_bridge, start_link, [Name, Config]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_bridge]}. -%% @doc List all bridges -spec(bridges() -> [{node(), map()}]). bridges() -> - [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?MODULE)]. + [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. -init([]) -> - BridgesOpts = emqx_config:get_env(bridges, []), - Bridges = [spec(Opts)|| Opts <- BridgesOpts], - {ok, {{one_for_one, 10, 100}, Bridges}}. +create_bridge(Id, Config) -> + supervisor:start_child(?SUP, bridge_spec({Id, Config})). -spec({Id, Options})-> - #{id => Id, - start => {emqx_bridge, start_link, [Id, Options]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_bridge]}. +drop_bridge(Id) -> + case supervisor:terminate_child(?SUP, Id) of + ok -> + supervisor:delete_child(?SUP, Id); + Error -> + ?LOG(error, "[Bridge] Delete bridge failed", [Error]), + Error + end. diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 7ebd40769..2275164a9 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -18,6 +18,7 @@ -include("types.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_client.hrl"). -export([start_link/0, start_link/1]). -export([request/5, request/6, request_async/7, receive_response/3]). @@ -37,12 +38,12 @@ %% For test cases -export([pause/1, resume/1]). --export([initialized/3, waiting_for_connack/3, connected/3]). +-export([initialized/3, waiting_for_connack/3, connected/3, inflight_full/3]). -export([init/1, callback_mode/0, handle_event/4, terminate/3, code_change/4]). -export_type([client/0, properties/0, payload/0, pubopt/0, subopt/0, request_input/0, response_payload/0, request_handler/0, - corr_data/0]). + corr_data/0, mqtt_msg/0]). -export_type([host/0, option/0]). @@ -58,7 +59,7 @@ -define(RESPONSE_TIMEOUT_SECONDS, timer:seconds(5)). --define(NO_HANDLER, undefined). +-define(NO_REQ_HANDLER, undefined). -define(NO_GROUP, <<>>). @@ -66,10 +67,23 @@ -type(host() :: inet:ip_address() | inet:hostname()). --type corr_data() :: binary(). +-type(corr_data() :: binary()). + +%% NOTE: Message handler is different from request handler. +%% Message handler is a set of callbacks defined to handle MQTT messages as well as +%% the disconnect event. +%% Request handler is a callback to handle received MQTT message as in 'request', +%% and publish another MQTT message back to the defined topic as in 'response'. +%% `owner' and `msg_handler' has no effect when `request_handler' is set. +-define(NO_MSG_HDLR, undefined). +-type(msg_handler() :: #{puback := fun((_) -> any()), + publish := fun((emqx_types:message()) -> any()), + disconnected := fun(({reason_code(), _Properties :: term()}) -> any()) + }). -type(option() :: {name, atom()} | {owner, pid()} + | {msg_handler, msg_handler()} | {host, host()} | {hosts, [{host(), inet:port_number()}]} | {port, inet:port_number()} @@ -97,13 +111,11 @@ | {force_ping, boolean()} | {properties, properties()}). --record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, - packet_id, topic, props, payload}). - -type(mqtt_msg() :: #mqtt_msg{}). -record(state, {name :: atom(), owner :: pid(), + msg_handler :: ?NO_MSG_HDLR | msg_handler(), host :: host(), port :: inet:port_number(), hosts :: [{host(), inet:port_number()}], @@ -378,7 +390,7 @@ publish(Client, Topic, Properties, Payload, Opts) payload = iolist_to_binary(Payload)}). -spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}). -publish(Client, Msg) when is_record(Msg, mqtt_msg) -> +publish(Client, Msg) -> gen_statem:call(Client, {publish, Msg}). -spec(unsubscribe(client(), topic() | [topic()]) -> subscribe_ret()). @@ -499,7 +511,7 @@ init([Options]) -> auto_ack = true, ack_timeout = ?DEFAULT_ACK_TIMEOUT, retry_interval = 0, - request_handler = ?NO_HANDLER, + request_handler = ?NO_REQ_HANDLER, connect_timeout = ?DEFAULT_CONNECT_TIMEOUT, last_packet_id = 1}), {ok, initialized, init_parse_state(State)}. @@ -518,6 +530,8 @@ init([{name, Name} | Opts], State) -> init([{owner, Owner} | Opts], State) when is_pid(Owner) -> link(Owner), init(Opts, State#state{owner = Owner}); +init([{msg_handler, Hdlr} | Opts], State) -> + init(Opts, State#state{msg_handler = Hdlr}); init([{host, Host} | Opts], State) -> init(Opts, State#state{host = Host}); init([{port, Port} | Opts], State) -> @@ -729,12 +743,12 @@ waiting_for_connack(EventType, EventContent, State) -> false -> {stop, connack_timeout} end. -connected({call, From}, subscriptions, State = #state{subscriptions = Subscriptions}) -> - {keep_state, State, [{reply, From, maps:to_list(Subscriptions)}]}; +connected({call, From}, subscriptions, #state{subscriptions = Subscriptions}) -> + {keep_state_and_data, [{reply, From, maps:to_list(Subscriptions)}]}; connected({call, From}, info, State) -> Info = lists:zip(record_info(fields, state), tl(tuple_to_list(State))), - {keep_state, State, [{reply, From, Info}]}; + {keep_state_and_data, [{reply, From, Info}]}; connected({call, From}, pause, State) -> {keep_state, State#state{paused = true}, [{reply, From, ok}]}; @@ -742,14 +756,11 @@ connected({call, From}, pause, State) -> connected({call, From}, resume, State) -> {keep_state, State#state{paused = false}, [{reply, From, ok}]}; -connected({call, From}, stop, _State) -> - {stop_and_reply, normal, [{reply, From, ok}]}; +connected({call, From}, get_properties, #state{properties = Properties}) -> + {keep_state_and_data, [{reply, From, Properties}]}; -connected({call, From}, get_properties, State = #state{properties = Properties}) -> - {keep_state, State, [{reply, From, Properties}]}; - -connected({call, From}, client_id, State = #state{client_id = ClientId}) -> - {keep_state, State, [{reply, From, ClientId}]}; +connected({call, From}, client_id, #state{client_id = ClientId}) -> + {keep_state_and_data, [{reply, From, ClientId}]}; connected({call, From}, {set_request_handler, RequestHandler}, State) -> {keep_state, State#state{request_handler = RequestHandler}, [{reply, From, ok}]}; @@ -779,19 +790,18 @@ connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) -> connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, State = #state{inflight = Inflight, last_packet_id = PacketId}) when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> - case emqx_inflight:is_full(Inflight) of - true -> - {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]}; - false -> - Msg1 = Msg#mqtt_msg{packet_id = PacketId}, - case send(Msg1, State) of - {ok, NewState} -> - Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), - {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), - [{reply, From, {ok, PacketId}}]}; - {error, Reason} -> - {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} - end + Msg1 = Msg#mqtt_msg{packet_id = PacketId}, + case send(Msg1, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), + State1 = ensure_retry_timer(NewState#state{inflight = Inflight1}), + Actions = [{reply, From, {ok, PacketId}}], + case emqx_inflight:is_full(Inflight1) of + true -> {next_state, inflight_full, State1, Actions}; + false -> {keep_state, State1, Actions} + end; + {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} end; connected({call, From}, UnsubReq = {unsubscribe, Properties, Topics}, @@ -833,8 +843,8 @@ connected(cast, {pubrel, PacketId, ReasonCode, Properties}, State) -> connected(cast, {pubcomp, PacketId, ReasonCode, Properties}, State) -> send_puback(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), State); -connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), State = #state{paused = true}) -> - {keep_state, State}; +connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), #state{paused = true}) -> + keep_state_and_data; connected(cast, Packet = ?PUBLISH_PACKET(?QOS_0, _Topic, _PacketId, Properties, Payload), State) when Properties =/= undefined -> @@ -858,18 +868,8 @@ connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _Topic, _PacketId, Properties, connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> publish_process(?QOS_2, Packet, State); -connected(cast, ?PUBACK_PACKET(PacketId, ReasonCode, Properties), - State = #state{owner = Owner, inflight = Inflight}) -> - case emqx_inflight:lookup(PacketId, Inflight) of - {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} -> - Owner ! {puback, #{packet_id => PacketId, - reason_code => ReasonCode, - properties => Properties}}, - {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; - none -> - emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]), - {keep_state, State} - end; +connected(cast, ?PUBACK_PACKET(_PacketId, _ReasonCode, _Properties) = PubAck, State) -> + {keep_state, delete_inflight(PubAck, State)}; connected(cast, ?PUBREC_PACKET(PacketId), State = #state{inflight = Inflight}) -> send_puback(?PUBREL_PACKET(PacketId), @@ -897,21 +897,11 @@ connected(cast, ?PUBREL_PACKET(PacketId), end; error -> emqx_logger:warning("Unexpected PUBREL: ~p", [PacketId]), - {keep_state, State} + keep_state_and_data end; -connected(cast, ?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), - State = #state{owner = Owner, inflight = Inflight}) -> - case emqx_inflight:lookup(PacketId, Inflight) of - {value, {pubrel, _PacketId, _Ts}} -> - Owner ! {puback, #{packet_id => PacketId, - reason_code => ReasonCode, - properties => Properties}}, - {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; - none -> - emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]), - {keep_state, State} - end; +connected(cast, ?PUBCOMP_PACKET(_PacketId, _ReasonCode, _Properties) = PubComp, State) -> + {keep_state, delete_inflight(PubComp, State)}; connected(cast, ?SUBACK_PACKET(PacketId, Properties, ReasonCodes), State = #state{subscriptions = _Subscriptions}) -> @@ -920,7 +910,8 @@ connected(cast, ?SUBACK_PACKET(PacketId, Properties, ReasonCodes), %%TODO: Merge reason codes to subscriptions? Reply = {ok, Properties, ReasonCodes}, {keep_state, NewState, [{reply, From, Reply}]}; - false -> {keep_state, State} + false -> + keep_state_and_data end; connected(cast, ?UNSUBACK_PACKET(PacketId, Properties, ReasonCodes), @@ -933,22 +924,22 @@ connected(cast, ?UNSUBACK_PACKET(PacketId, Properties, ReasonCodes), end, Subscriptions, Topics), {keep_state, NewState#state{subscriptions = Subscriptions1}, [{reply, From, {ok, Properties, ReasonCodes}}]}; - false -> {keep_state, State} + false -> + keep_state_and_data end; -connected(cast, ?PACKET(?PINGRESP), State = #state{pending_calls = []}) -> - {keep_state, State}; +connected(cast, ?PACKET(?PINGRESP), #state{pending_calls = []}) -> + keep_state_and_data; connected(cast, ?PACKET(?PINGRESP), State) -> case take_call(ping, State) of {value, #call{from = From}, NewState} -> {keep_state, NewState, [{reply, From, pong}]}; - false -> {keep_state, State} + false -> + keep_state_and_data end; -connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), - State = #state{owner = Owner}) -> - Owner ! {disconnected, ReasonCode, Properties}, - {stop, disconnected, State}; +connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), State) -> + {stop, {disconnected, ReasonCode, Properties}, State}; connected(info, {timeout, _TRef, keepalive}, State = #state{force_ping = true}) -> case send(?PACKET(?PINGREQ), State) of @@ -989,15 +980,19 @@ connected(info, {timeout, TRef, retry}, State = #state{retry_timer = TRef, connected(EventType, EventContent, Data) -> handle_event(EventType, EventContent, connected, Data). -should_ping(Sock) -> - case emqx_client_sock:getstat(Sock, [send_oct]) of - {ok, [{send_oct, Val}]} -> - OldVal = get(send_oct), put(send_oct, Val), - OldVal == undefined orelse OldVal == Val; - Error = {error, _Reason} -> - Error - end. +inflight_full({call, _From}, {publish, #mqtt_msg{qos = QoS}}, _State) when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> + {keep_state_and_data, [postpone]}; +inflight_full(cast, ?PUBACK_PACKET(_PacketId, _ReasonCode, _Properties) = PubAck, State) -> + delete_inflight_when_full(PubAck, State); +inflight_full(cast, ?PUBCOMP_PACKET(_PacketId, _ReasonCode, _Properties) = PubComp, State) -> + delete_inflight_when_full(PubComp, State); +inflight_full(EventType, EventContent, Data) -> + %% inflight_full is a sub-state of connected state, + %% delegate all other events to connected state. + connected(EventType, EventContent, Data). +handle_event({call, From}, stop, _StateName, _State) -> + {stop_and_reply, normal, [{reply, From, ok}]}; handle_event(info, {TcpOrSsL, _Sock, Data}, _StateName, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> emqx_logger:debug("RECV Data: ~p", [Data]), @@ -1017,23 +1012,31 @@ handle_event(info, {'EXIT', Owner, Reason}, _, State = #state{owner = Owner}) -> emqx_logger:debug("[~p] Got EXIT from owner, Reason: ~p", [?MODULE, Reason]), {stop, {shutdown, Reason}, State}; -handle_event(info, {inet_reply, _Sock, ok}, _, State) -> - {keep_state, State}; +handle_event(info, {inet_reply, _Sock, ok}, _, _State) -> + keep_state_and_data; handle_event(info, {inet_reply, _Sock, {error, Reason}}, _, State) -> emqx_logger:error("[~p] got tcp error: ~p", [?MODULE, Reason]), {stop, {shutdown, Reason}, State}; -handle_event(EventType, EventContent, StateName, StateData) -> +handle_event(EventType, EventContent, StateName, _StateData) -> emqx_logger:error("State: ~s, Unexpected Event: (~p, ~p)", [StateName, EventType, EventContent]), - {keep_state, StateData}. + keep_state_and_data. %% Mandatory callback functions -terminate(_Reason, _State, #state{socket = undefined}) -> - ok; -terminate(_Reason, _State, #state{socket = Socket}) -> - emqx_client_sock:close(Socket). +terminate(Reason, _StateName, State = #state{socket = Socket}) -> + case Reason of + {disconnected, ReasonCode, Properties} -> + %% backward compatible + ok = eval_msg_handler(State, disconnected, {ReasonCode, Properties}); + _ -> + ok = eval_msg_handler(State, disconnected, Reason) + end, + case Socket =:= undefined of + true -> ok; + _ -> emqx_client_sock:close(Socket) + end. code_change(_Vsn, State, Data, _Extra) -> {ok, State, Data}. @@ -1042,6 +1045,47 @@ code_change(_Vsn, State, Data, _Extra) -> %% Internal functions %%------------------------------------------------------------------------------ +should_ping(Sock) -> + case emqx_client_sock:getstat(Sock, [send_oct]) of + {ok, [{send_oct, Val}]} -> + OldVal = get(send_oct), put(send_oct, Val), + OldVal == undefined orelse OldVal == Val; + Error = {error, _Reason} -> + Error + end. + +delete_inflight(?PUBACK_PACKET(PacketId, ReasonCode, Properties), + State = #state{inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} -> + ok = eval_msg_handler(State, puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}), + State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}; + none -> + emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]), + State + end; +delete_inflight(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), + State = #state{inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {pubrel, _PacketId, _Ts}} -> + ok = eval_msg_handler(State, puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}), + State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}; + none -> + emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]), + State + end. + +delete_inflight_when_full(Packet, State0) -> + State = #state{inflight = Inflight} = delete_inflight(Packet, State0), + case emqx_inflight:is_full(Inflight) of + true -> {keep_state, State}; + false -> {next_state, connected, State} + end. + %% Subscribe to response topic. -spec(sub_response_topic(client(), qos(), topic()) -> ok). sub_response_topic(Client, QoS, Topic) when is_binary(Topic) -> @@ -1103,8 +1147,8 @@ assign_id(?NO_CLIENT_ID, Props) -> assign_id(Id, _Props) -> Id. -publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State = #state{auto_ack = AutoAck}) -> - _ = deliver(packet_to_msg(Packet), State), +publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State0 = #state{auto_ack = AutoAck}) -> + State = deliver(packet_to_msg(Packet), State0), case AutoAck of true -> send_puback(?PUBACK_PACKET(PacketId), State); false -> {keep_state, State} @@ -1118,18 +1162,11 @@ publish_process(?QOS_2, Packet = ?PUBLISH_PACKET(?QOS_2, PacketId), Stop -> Stop end. -response_publish(undefined, State, _QoS, _Payload) -> - State; -response_publish(Properties, State = #state{request_handler = RequestHandler}, QoS, Payload) -> - case maps:find('Response-Topic', Properties) of - {ok, ResponseTopic} -> - case RequestHandler of - ?NO_HANDLER -> State; - _ -> do_publish(ResponseTopic, Properties, State, QoS, Payload) - end; - _ -> - State - end. +response_publish(#{'Response-Topic' := ResponseTopic} = Properties, + State = #state{request_handler = RequestHandler}, QoS, Payload) + when RequestHandler =/= ?NO_REQ_HANDLER -> + do_publish(ResponseTopic, Properties, State, QoS, Payload); +response_publish(_Properties, State, _QoS, _Payload) -> State. do_publish(ResponseTopic, Properties, State = #state{request_handler = RequestHandler}, ?QOS_0, Payload) -> Msg = #mqtt_msg{qos = ?QOS_0, @@ -1210,11 +1247,12 @@ ensure_ack_timer(State = #state{ack_timer = undefined, ensure_ack_timer(State) -> State. ensure_retry_timer(State = #state{retry_interval = Interval}) -> - ensure_retry_timer(Interval, State). -ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) + do_ensure_retry_timer(Interval, State). + +do_ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) when Interval > 0 -> State#state{retry_timer = erlang:start_timer(Interval, self(), retry)}; -ensure_retry_timer(_Interval, State) -> +do_ensure_retry_timer(_Interval, State) -> State. retry_send(State = #state{inflight = Inflight}) -> @@ -1231,7 +1269,7 @@ retry_send([{Type, Msg, Ts} | Msgs], Now, State = #state{retry_interval = Interv {ok, NewState} -> retry_send(Msgs, Now, NewState); {error, Error} -> {stop, Error} end; - false -> {keep_state, ensure_retry_timer(Interval - Diff, State)} + false -> {keep_state, do_ensure_retry_timer(Interval - Diff, State)} end. retry_send(publish, Msg = #mqtt_msg{qos = QoS, packet_id = PacketId}, @@ -1253,19 +1291,37 @@ retry_send(pubrel, PacketId, Now, State = #state{inflight = Inflight}) -> Error end. +deliver(_Msg, State = #state{request_handler = Hdlr}) when Hdlr =/= ?NO_REQ_HANDLER -> + %% message has been terminated by request handler, hence should not continue processing + State; deliver(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId, topic = Topic, props = Props, payload = Payload}, - State = #state{owner = Owner, request_handler = RequestHandler}) -> - case RequestHandler of - ?NO_HANDLER -> - Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId, - topic => Topic, properties => Props, payload => Payload, - client_pid => self()}}; - _ -> - ok - end, + State) -> + Msg = #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId, + topic => Topic, properties => Props, payload => Payload, + client_pid => self()}, + ok = eval_msg_handler(State, publish, Msg), State. +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, + owner = Owner}, + disconnected, {ReasonCode, Properties}) -> + %% Special handling for disconnected message when there is no handler callback + Owner ! {disconnected, ReasonCode, Properties}, + ok; +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER}, + disconnected, _OtherReason) -> + %% do nothing to be backward compatible + ok; +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, + owner = Owner}, Kind, Msg) -> + Owner ! {Kind, Msg}, + ok; +eval_msg_handler(#state{msg_handler = Handler}, Kind, Msg) -> + F = maps:get(Kind, Handler), + _ = F(Msg), + ok. + packet_to_msg(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, dup = Dup, qos = QoS, @@ -1319,9 +1375,9 @@ send(Msg, State) when is_record(Msg, mqtt_msg) -> send(Packet, State = #state{socket = Sock, proto_ver = Ver}) when is_record(Packet, mqtt_packet) -> Data = emqx_frame:serialize(Packet, #{version => Ver}), - emqx_logger:debug("SEND Data: ~p", [Data]), + emqx_logger:debug("SEND Data: ~1000p", [Packet]), case emqx_client_sock:send(Sock, Data) of - ok -> {ok, next_packet_id(State)}; + ok -> {ok, bump_last_packet_id(State)}; Error -> Error end. @@ -1355,10 +1411,11 @@ next_events(Packets) -> [{next_event, cast, Packet} || Packet <- lists:reverse(Packets)]. %%------------------------------------------------------------------------------ -%% Next packet id +%% packet_id generation -next_packet_id(State = #state{last_packet_id = 16#ffff}) -> - State#state{last_packet_id = 1}; +bump_last_packet_id(State = #state{last_packet_id = Id}) -> + State#state{last_packet_id = next_packet_id(Id)}. -next_packet_id(State = #state{last_packet_id = Id}) -> - State#state{last_packet_id = Id + 1}. +-spec next_packet_id(packet_id()) -> packet_id(). +next_packet_id(?MAX_PACKET_ID) -> 1; +next_packet_id(Id) -> Id + 1. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index da75d211e..e30958735 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -14,20 +14,22 @@ -module(emqx_connection). --behaviour(gen_server). +-behaviour(gen_statem). -include("emqx.hrl"). -include("emqx_mqtt.hrl"). -include("logger.hrl"). -export([start_link/3]). --export([info/1, attrs/1, stats/1]). +-export([info/1]). +-export([attrs/1]). +-export([stats/1]). -export([kick/1]). -export([session/1]). -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +%% gen_statem callbacks +-export([idle/3, connected/3]). +-export([init/1, callback_mode/0, code_change/4, terminate/3]). -record(state, { transport, @@ -37,7 +39,7 @@ conn_state, active_n, proto_state, - parser_state, + parse_state, gc_state, keepalive, enable_stats, @@ -48,28 +50,29 @@ idle_timeout }). --define(DEFAULT_ACTIVE_N, 100). +-define(ACTIVE_N, 100). +-define(HANDLE(T, C, D), handle((T), (C), (D))). -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). start_link(Transport, Socket, Options) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Socket, Options]])}. + {ok, proc_lib:spawn_link(?MODULE, init, [{Transport, Socket, Options}])}. %%------------------------------------------------------------------------------ %% API %%------------------------------------------------------------------------------ -%% for debug +%% For debug info(CPid) when is_pid(CPid) -> call(CPid, info); -info(#state{transport = Transport, - socket = Socket, - peername = Peername, - sockname = Sockname, - conn_state = ConnState, - active_n = ActiveN, - rate_limit = RateLimit, - pub_limit = PubLimit, +info(#state{transport = Transport, + socket = Socket, + peername = Peername, + sockname = Sockname, + conn_state = ConnState, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, proto_state = ProtoState}) -> ConnInfo = [{socktype, Transport:type(Socket)}, {peername, Peername}, @@ -81,10 +84,12 @@ info(#state{transport = Transport, ProtoInfo = emqx_protocol:info(ProtoState), lists:usort(lists:append(ConnInfo, ProtoInfo)). -rate_limit_info(undefined) -> #{}; -rate_limit_info(Limit) -> esockd_rate_limit:info(Limit). +rate_limit_info(undefined) -> + #{}; +rate_limit_info(Limit) -> + esockd_rate_limit:info(Limit). -%% for dashboard +%% For dashboard attrs(CPid) when is_pid(CPid) -> call(CPid, attrs); @@ -100,277 +105,306 @@ attrs(#state{peername = Peername, stats(CPid) when is_pid(CPid) -> call(CPid, stats); -stats(#state{transport = Transport, - socket = Socket, +stats(#state{transport = Transport, + socket = Socket, proto_state = ProtoState}) -> - lists:append([emqx_misc:proc_stats(), - emqx_protocol:stats(ProtoState), - case Transport:getstat(Socket, ?SOCK_STATS) of - {ok, Ss} -> Ss; - {error, _} -> [] - end]). + SockStats = case Transport:getstat(Socket, ?SOCK_STATS) of + {ok, Ss} -> Ss; + {error, _} -> [] + end, + lists:append([SockStats, + emqx_misc:proc_stats(), + emqx_protocol:stats(ProtoState)]). -kick(CPid) -> call(CPid, kick). +kick(CPid) -> + call(CPid, kick). -session(CPid) -> call(CPid, session). +session(CPid) -> + call(CPid, session). call(CPid, Req) -> - gen_server:call(CPid, Req, infinity). + gen_statem:call(CPid, Req, infinity). %%------------------------------------------------------------------------------ -%% gen_server callbacks +%% gen_statem callbacks %%------------------------------------------------------------------------------ -init([Transport, RawSocket, Options]) -> - case Transport:wait(RawSocket) of - {ok, Socket} -> - Zone = proplists:get_value(zone, Options), - {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), - {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), - Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), - RateLimit = init_limiter(proplists:get_value(rate_limit, Options)), - PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)), - ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N), - EnableStats = emqx_zone:get_env(Zone, enable_stats, true), - IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), - SendFun = send_fun(Transport, Socket), - ProtoState = emqx_protocol:init(#{peername => Peername, - sockname => Sockname, - peercert => Peercert, - sendfun => SendFun}, Options), - ParserState = emqx_protocol:parser(ProtoState), - GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), - GcState = emqx_gc:init(GcPolicy), - State = run_socket(#state{transport = Transport, - socket = Socket, - peername = Peername, - conn_state = running, - active_n = ActiveN, - rate_limit = RateLimit, - pub_limit = PubLimit, - proto_state = ProtoState, - parser_state = ParserState, - gc_state = GcState, - enable_stats = EnableStats, - idle_timeout = IdleTimout - }), - ok = emqx_misc:init_proc_mng_policy(Zone), - emqx_logger:set_metadata_peername(esockd_net:format(Peername)), - gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], - State, self(), IdleTimout); - {error, Reason} -> - {stop, Reason} - end. +init({Transport, RawSocket, Options}) -> + {ok, Socket} = Transport:wait(RawSocket), + {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), + {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), + Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), + emqx_logger:set_metadata_peername(esockd_net:format(Peername)), + Zone = proplists:get_value(zone, Options), + RateLimit = init_limiter(proplists:get_value(rate_limit, Options)), + PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)), + ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N), + EnableStats = emqx_zone:get_env(Zone, enable_stats, true), + IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), + SendFun = fun(Data) -> Transport:async_send(Socket, Data) end, + ProtoState = emqx_protocol:init(#{peername => Peername, + sockname => Sockname, + peercert => Peercert, + sendfun => SendFun, + conn_mod => ?MODULE}, Options), + ParseState = emqx_protocol:parser(ProtoState), + GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), + GcState = emqx_gc:init(GcPolicy), + State = #state{transport = Transport, + socket = Socket, + peername = Peername, + conn_state = running, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState, + parse_state = ParseState, + gc_state = GcState, + enable_stats = EnableStats, + idle_timeout = IdleTimout}, + ok = emqx_misc:init_proc_mng_policy(Zone), + gen_statem:enter_loop(?MODULE, [{hibernate_after, 2 * IdleTimout}], + idle, State, self(), [IdleTimout]). init_limiter(undefined) -> undefined; init_limiter({Rate, Burst}) -> esockd_rate_limit:new(Rate, Burst). -send_fun(Transport, Socket) -> - fun(Packet, Options) -> - Data = emqx_frame:serialize(Packet, Options), - try Transport:async_send(Socket, Data) of - ok -> - emqx_metrics:trans(inc, 'bytes/sent', iolist_size(Data)), - ok; - Error -> Error - catch - error:Error -> - {error, Error} - end - end. +callback_mode() -> + [state_functions, state_enter]. -handle_call(info, _From, State) -> - {reply, info(State), State}; +%%------------------------------------------------------------------------------ +%% Idle state -handle_call(attrs, _From, State) -> - {reply, attrs(State), State}; +idle(enter, _, State) -> + ok = activate_socket(State), + keep_state_and_data; -handle_call(stats, _From, State) -> - {reply, stats(State), State}; +idle(timeout, _Timeout, State) -> + {stop, idle_timeout, State}; -handle_call(kick, _From, State) -> - {stop, {shutdown, kicked}, ok, State}; +idle(cast, {incoming, Packet, PState}, _State) -> + handle_packet(Packet, fun(NState) -> + {next_state, connected, reset_parser(NState)} + end, PState); -handle_call(session, _From, State = #state{proto_state = ProtoState}) -> - {reply, emqx_protocol:session(ProtoState), State}; +idle(EventType, Content, State) -> + ?HANDLE(EventType, Content, State). -handle_call(Req, _From, State) -> - ?LOG(error, "unexpected call: ~p", [Req]), - {reply, ignored, State}. +%%------------------------------------------------------------------------------ +%% Connected state -handle_cast(Msg, State) -> - ?LOG(error, "unexpected cast: ~p", [Msg]), - {noreply, State}. +connected(enter, _, _State) -> + %% What to do? + keep_state_and_data; -handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> +%% Handle Input +connected(cast, {incoming, Packet = ?PACKET(Type), PState}, _State) -> + _ = emqx_metrics:received(Packet), + (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1), + handle_packet(Packet, fun(NState) -> + {keep_state, reset_parser(NState)} + end, PState); + +%% Handle Output +connected(info, {deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> case emqx_protocol:deliver(PubOrAck, ProtoState) of - {ok, ProtoState1} -> - State1 = State#state{proto_state = ProtoState1}, - {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))}; + {ok, NProtoState} -> + NState = State#state{proto_state = NProtoState}, + {keep_state, maybe_gc(PubOrAck, NState)}; {error, Reason} -> shutdown(Reason, State) end; -handle_info({timeout, Timer, emit_stats}, - State = #state{stats_timer = Timer, - proto_state = ProtoState, - gc_state = GcState}) -> - emqx_metrics:commit(), - emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), - NewState = State#state{stats_timer = undefined}, - Limits = erlang:get(force_shutdown_policy), - case emqx_misc:conn_proc_mng_policy(Limits) of - continue -> - {noreply, NewState}; - hibernate -> - %% going to hibernate, reset gc stats - GcState1 = emqx_gc:reset(GcState), - {noreply, NewState#state{gc_state = GcState1}, hibernate}; - {shutdown, Reason} -> - ?LOG(warning, "shutdown due to ~p", [Reason]), - shutdown(Reason, NewState) - end; - -handle_info(timeout, State) -> - shutdown(idle_timeout, State); - -handle_info({shutdown, Reason}, State) -> - shutdown(Reason, State); - -handle_info({shutdown, discard, {ClientId, ByPid}}, State) -> - ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]), - shutdown(discard, State); - -handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> - ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), - shutdown(conflict, State); - -handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> - process_incoming(Data, State); - -%% Rate limit here, cool:) -handle_info({tcp_passive, _Sock}, State) -> - {noreply, run_socket(ensure_rate_limit(State))}; -%% FIXME Later -handle_info({ssl_passive, _Sock}, State) -> - {noreply, run_socket(ensure_rate_limit(State))}; - -handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error -> - shutdown(Reason, State); - -handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed -> - shutdown(closed, State); - -%% Rate limit timer -handle_info(activate_sock, State) -> - {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; - -handle_info({inet_reply, _Sock, ok}, State) -> - {noreply, State}; - -handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({keepalive, start, Interval}, State = #state{transport = Transport, socket = Socket}) -> - ?LOG(debug, "Keepalive at the interval of ~p", [Interval]), +%% Start Keepalive +connected(info, {keepalive, start, Interval}, + State = #state{transport = Transport, socket = Socket}) -> StatFun = fun() -> case Transport:getstat(Socket, [recv_oct]) of {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; - Error -> Error + Error -> Error end - end, + end, case emqx_keepalive:start(StatFun, Interval, {keepalive, check}) of {ok, KeepAlive} -> - {noreply, State#state{keepalive = KeepAlive}}; + {keep_state, State#state{keepalive = KeepAlive}}; {error, Error} -> shutdown(Error, State) end; -handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> +%% Keepalive timer +connected(info, {keepalive, check}, State = #state{keepalive = KeepAlive}) -> case emqx_keepalive:check(KeepAlive) of {ok, KeepAlive1} -> - {noreply, State#state{keepalive = KeepAlive1}}; + {keep_state, State#state{keepalive = KeepAlive1}}; {error, timeout} -> shutdown(keepalive_timeout, State); {error, Error} -> shutdown(Error, State) end; -handle_info(Info, State) -> - ?LOG(error, "unexpected info: ~p", [Info]), - {noreply, State}. +connected(EventType, Content, State) -> + ?HANDLE(EventType, Content, State). -terminate(Reason, #state{transport = Transport, - socket = Socket, - keepalive = KeepAlive, - proto_state = ProtoState}) -> +%% Handle call +handle({call, From}, info, State) -> + reply(From, info(State), State); + +handle({call, From}, attrs, State) -> + reply(From, attrs(State), State); + +handle({call, From}, stats, State) -> + reply(From, stats(State), State); + +handle({call, From}, kick, State) -> + ok = gen_statem:reply(From, ok), + shutdown(kicked, State); + +handle({call, From}, session, State = #state{proto_state = ProtoState}) -> + reply(From, emqx_protocol:session(ProtoState), State); + +handle({call, From}, Req, State) -> + ?LOG(error, "unexpected call: ~p", [Req]), + reply(From, ignored, State); + +%% Handle cast +handle(cast, Msg, State) -> + ?LOG(error, "unexpected cast: ~p", [Msg]), + {keep_state, State}; + +%% Handle Incoming +handle(info, {Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> + Oct = iolist_size(Data), + ?LOG(debug, "RECV ~p", [Data]), + emqx_pd:update_counter(incoming_bytes, Oct), + emqx_metrics:trans(inc, 'bytes/received', Oct), + NState = ensure_stats_timer(maybe_gc({1, Oct}, State)), + process_incoming(Data, [], NState); + +handle(info, {Error, _Sock, Reason}, State) + when Error == tcp_error; Error == ssl_error -> + shutdown(Reason, State); + +handle(info, {Closed, _Sock}, State) + when Closed == tcp_closed; Closed == ssl_closed -> + shutdown(closed, State); + +handle(info, {tcp_passive, _Sock}, State) -> + %% Rate limit here:) + NState = ensure_rate_limit(State), + ok = activate_socket(NState), + {keep_state, NState}; + +handle(info, activate_socket, State) -> + %% Rate limit timer expired. + ok = activate_socket(State), + {keep_state, State#state{conn_state = running, limit_timer = undefined}}; + +handle(info, {inet_reply, _Sock, ok}, State) -> + %% something sent + {keep_state, ensure_stats_timer(State)}; + +handle(info, {inet_reply, _Sock, {error, Reason}}, State) -> + shutdown(Reason, State); + +handle(info, {timeout, Timer, emit_stats}, + State = #state{stats_timer = Timer, + proto_state = ProtoState, + gc_state = GcState}) -> + emqx_metrics:commit(), + emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), + NState = State#state{stats_timer = undefined}, + Limits = erlang:get(force_shutdown_policy), + case emqx_misc:conn_proc_mng_policy(Limits) of + continue -> + {keep_state, NState}; + hibernate -> + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {keep_state, NState#state{gc_state = GcState1}, hibernate}; + {shutdown, Reason} -> + ?LOG(warning, "shutdown due to ~p", [Reason]), + shutdown(Reason, NState) + end; + +handle(info, {shutdown, discard, {ClientId, ByPid}}, State) -> + ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]), + shutdown(discard, State); + +handle(info, {shutdown, conflict, {ClientId, NewPid}}, State) -> + ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), + shutdown(conflict, State); + +handle(info, {shutdown, Reason}, State) -> + shutdown(Reason, State); + +handle(info, Info, State) -> + ?LOG(error, "unexpected info: ~p", [Info]), + {keep_state, State}. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +terminate(Reason, _StateName, #state{transport = Transport, + socket = Socket, + keepalive = KeepAlive, + proto_state = ProtoState}) -> ?LOG(debug, "Terminated for ~p", [Reason]), Transport:fast_close(Socket), emqx_keepalive:cancel(KeepAlive), case {ProtoState, Reason} of {undefined, _} -> ok; {_, {shutdown, Error}} -> - emqx_protocol:shutdown(Error, ProtoState); + emqx_protocol:terminate(Error, ProtoState); {_, Reason} -> - emqx_protocol:shutdown(Reason, ProtoState) + emqx_protocol:terminate(Reason, ProtoState) end. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%------------------------------------------------------------------------------ -%% Internals: process incoming, parse and handle packets %%------------------------------------------------------------------------------ +%% Process incoming data -process_incoming(Data, State) -> - Oct = iolist_size(Data), - ?LOG(debug, "RECV ~p", [Data]), - emqx_pd:update_counter(incoming_bytes, Oct), - emqx_metrics:trans(inc, 'bytes/received', Oct), - case handle_packet(Data, State) of - {noreply, State1} -> - State2 = maybe_gc({1, Oct}, State1), - {noreply, ensure_stats_timer(State2)}; - Shutdown -> Shutdown - end. +process_incoming(<<>>, Packets, State) -> + {keep_state, State, next_events({Packets, State})}; -%% Parse and handle packets -handle_packet(<<>>, State) -> - {noreply, State}; - -handle_packet(Data, State = #state{proto_state = ProtoState, - parser_state = ParserState, - idle_timeout = IdleTimeout}) -> - try emqx_frame:parse(Data, ParserState) of - {more, ParserState1} -> - {noreply, State#state{parser_state = ParserState1}, IdleTimeout}; - {ok, Packet = ?PACKET(Type), Rest} -> - emqx_metrics:received(Packet), - (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1), - case emqx_protocol:received(Packet, ProtoState) of - {ok, ProtoState1} -> - handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1})); - {error, Reason} -> - ?LOG(error, "Process packet error - ~p", [Reason]), - shutdown(Reason, State); - {error, Reason, ProtoState1} -> - shutdown(Reason, State#state{proto_state = ProtoState1}); - {stop, Error, ProtoState1} -> - stop(Error, State#state{proto_state = ProtoState1}) - end; +process_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> + try emqx_frame:parse(Data, ParseState) of + {ok, Packet, Rest} -> + process_incoming(Rest, [Packet|Packets], reset_parser(State)); + {more, NewParseState} -> + {keep_state, State#state{parse_state = NewParseState}, next_events({Packets, State})}; {error, Reason} -> - ?LOG(error, "Parse frame error - ~p", [Reason]), shutdown(Reason, State) catch - _:Error -> - ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]), - shutdown(parse_error, State) + _:Error:Stk-> + ?LOG(error, "Parse failed for ~p~nStacktrace:~p~nError data:~p", [Error, Stk, Data]), + shutdown(Error, State) end. reset_parser(State = #state{proto_state = ProtoState}) -> - State#state{parser_state = emqx_protocol:parser(ProtoState)}. + State#state{parse_state = emqx_protocol:parser(ProtoState)}. + +next_events([]) -> + []; +next_events([{Packet, State}]) -> + {next_event, cast, {incoming, Packet, State}}; +next_events({Packets, State}) -> + [next_events([{Packet, State}]) || Packet <- lists:reverse(Packets)]. + +%%------------------------------------------------------------------------------ +%% Handle incoming packet + +handle_packet(Packet, SuccFun, State = #state{proto_state = ProtoState}) -> + case emqx_protocol:received(Packet, ProtoState) of + {ok, NProtoState} -> + SuccFun(State#state{proto_state = NProtoState}); + {error, Reason} -> + shutdown(Reason, State); + {error, Reason, NProtoState} -> + shutdown(Reason, State#state{proto_state = NProtoState}); + {stop, Error, NProtoState} -> + stop(Error, State#state{proto_state = NProtoState}) + end. %%------------------------------------------------------------------------------ %% Ensure rate limit @@ -389,27 +423,27 @@ ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) -> {0, Rl1} -> ensure_rate_limit(Limiters, setelement(Pos, State, Rl1)); {Pause, Rl1} -> - TRef = erlang:send_after(Pause, self(), activate_sock), + TRef = erlang:send_after(Pause, self(), activate_socket), setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1) end. %%------------------------------------------------------------------------------ %% Activate socket -run_socket(State = #state{conn_state = blocked}) -> - State; +activate_socket(#state{conn_state = blocked}) -> + ok; -run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) -> +activate_socket(#state{transport = Transport, socket = Socket, active_n = N}) -> TrueOrN = case Transport:is_ssl(Socket) of true -> true; %% Cannot set '{active, N}' for SSL:( false -> N end, - ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])), - State. - -ensure_ok_or_exit(ok) -> ok; -ensure_ok_or_exit({error, Reason}) -> - self() ! {shutdown, Reason}. + case Transport:setopts(Socket, [{active, TrueOrN}]) of + ok -> ok; + {error, Reason} -> + self() ! {shutdown, Reason}, + ok + end. %%------------------------------------------------------------------------------ %% Ensure stats timer @@ -418,6 +452,7 @@ ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined, idle_timeout = IdleTimeout}) -> State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; + ensure_stats_timer(State) -> State. %%------------------------------------------------------------------------------ @@ -425,20 +460,28 @@ ensure_stats_timer(State) -> State. maybe_gc(_, State = #state{gc_state = undefined}) -> State; -maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) -> +maybe_gc({publish, _, #message{payload = Payload}}, State) -> Oct = iolist_size(Payload), maybe_gc({1, Oct}, State); +maybe_gc(Packets, State) when is_list(Packets) -> + {Cnt, Oct} = + lists:unzip([{1, iolist_size(Payload)} + || {publish, _, #message{payload = Payload}} <- Packets]), + maybe_gc({lists:sum(Cnt), lists:sum(Oct)}, State); maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), State#state{gc_state = GCSt1}; -maybe_gc(_, State) -> - State. +maybe_gc(_, State) -> State. %%------------------------------------------------------------------------------ -%% Shutdown or stop +%% Helper functions + +reply(From, Reply, State) -> + {keep_state, State, [{reply, From, Reply}]}. shutdown(Reason, State) -> stop({shutdown, Reason}, State). stop(Reason, State) -> {stop, Reason, State}. + diff --git a/src/emqx_kernel_sup.erl b/src/emqx_kernel_sup.erl index bd98ccbdf..cfed226cd 100644 --- a/src/emqx_kernel_sup.erl +++ b/src/emqx_kernel_sup.erl @@ -26,7 +26,6 @@ start_link() -> init([]) -> {ok, {{one_for_one, 10, 100}, [child_spec(emqx_pool_sup, supervisor), - child_spec(emqx_alarm_mgr, worker), child_spec(emqx_hooks, worker), child_spec(emqx_stats, worker), child_spec(emqx_metrics, worker), diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl index 2a7375a7f..f52a376ba 100644 --- a/src/emqx_listeners.erl +++ b/src/emqx_listeners.erl @@ -56,7 +56,7 @@ start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> %% Start MQTT/WSS listener start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> - Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options), emqx_ws_connection, Options}]}]), + Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options), emqx_ws_connection, Options}]}]), start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, ranch_opts(Options), Dispatch). start_mqtt_listener(Name, ListenOn, Options) -> diff --git a/src/emqx_local_bridge.erl b/src/emqx_local_bridge.erl deleted file mode 100644 index 0521e6d3f..000000000 --- a/src/emqx_local_bridge.erl +++ /dev/null @@ -1,157 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_local_bridge). - --behaviour(gen_server). - --include("emqx.hrl"). --include("emqx_mqtt.hrl"). - --export([start_link/5]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --define(PING_DOWN_INTERVAL, 1000). - --record(state, {pool, id, - node, subtopic, - qos = ?QOS_0, - topic_suffix = <<>>, - topic_prefix = <<>>, - mqueue :: emqx_mqueue:mqueue(), - max_queue_len = 10000, - ping_down_interval = ?PING_DOWN_INTERVAL, - status = up}). - --type(option() :: {qos, emqx_mqtt_types:qos()} | - {topic_suffix, binary()} | - {topic_prefix, binary()} | - {max_queue_len, pos_integer()} | - {ping_down_interval, pos_integer()}). - --export_type([option/0]). - -%% @doc Start a bridge --spec(start_link(term(), pos_integer(), atom(), binary(), [option()]) - -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id, Node, Topic, Options) -> - gen_server:start_link(?MODULE, [Pool, Id, Node, Topic, Options], [{hibernate_after, 5000}]). - -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ - -init([Pool, Id, Node, Topic, Options]) -> - process_flag(trap_exit, true), - true = gproc_pool:connect_worker(Pool, {Pool, Id}), - case net_kernel:connect_node(Node) of - true -> - true = erlang:monitor_node(Node, true), - Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), - emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}), - State = parse_opts(Options, #state{node = Node, subtopic = Topic}), - MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len, - store_qos0 => true}), - {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}}; - false -> - {stop, {cannot_connect_node, Node}} - end. - -parse_opts([], State) -> - State; -parse_opts([{qos, QoS} | Opts], State) -> - parse_opts(Opts, State#state{qos = QoS}); -parse_opts([{topic_suffix, Suffix} | Opts], State) -> - parse_opts(Opts, State#state{topic_suffix= Suffix}); -parse_opts([{topic_prefix, Prefix} | Opts], State) -> - parse_opts(Opts, State#state{topic_prefix = Prefix}); -parse_opts([{max_queue_len, Len} | Opts], State) -> - parse_opts(Opts, State#state{max_queue_len = Len}); -parse_opts([{ping_down_interval, Interval} | Opts], State) -> - parse_opts(Opts, State#state{ping_down_interval = Interval}); -parse_opts([_Opt | Opts], State) -> - parse_opts(Opts, State). - -handle_call(Req, _From, State) -> - emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({dispatch, _Topic, Msg}, State = #state{mqueue = Q, status = down}) -> - %% TODO: how to drop??? - {_Dropped, NewQ} = emqx_mqueue:in(Msg, Q), - {noreply, State#state{mqueue = NewQ}}; - -handle_info({dispatch, _Topic, Msg}, State = #state{node = Node, status = up}) -> - emqx_rpc:cast(Node, emqx_broker, publish, [transform(Msg, State)]), - {noreply, State}; - -handle_info({nodedown, Node}, State = #state{node = Node, ping_down_interval = Interval}) -> - emqx_logger:warning("[Bridge] node down: ~s", [Node]), - erlang:send_after(Interval, self(), ping_down_node), - {noreply, State#state{status = down}, hibernate}; - -handle_info({nodeup, Node}, State = #state{node = Node}) -> - %% TODO: Really fast?? - case emqx:is_running(Node) of - true -> emqx_logger:warning("[Bridge] Node up: ~s", [Node]), - {noreply, dequeue(State#state{status = up})}; - false -> self() ! {nodedown, Node}, - {noreply, State#state{status = down}} - end; - -handle_info(ping_down_node, State = #state{node = Node, ping_down_interval = Interval}) -> - Self = self(), - spawn_link(fun() -> - case net_kernel:connect_node(Node) of - true -> Self ! {nodeup, Node}; - false -> erlang:send_after(Interval, Self, ping_down_node) - end - end), - {noreply, State}; - -handle_info({'EXIT', _Pid, normal}, State) -> - {noreply, State}; - -handle_info(Info, State) -> - emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - gproc_pool:disconnect_worker(Pool, {Pool, Id}). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -dequeue(State = #state{mqueue = MQ}) -> - case emqx_mqueue:out(MQ) of - {empty, MQ1} -> - State#state{mqueue = MQ1}; - {{value, Msg}, MQ1} -> - handle_info({dispatch, Msg#message.topic, Msg}, State), - dequeue(State#state{mqueue = MQ1}) - end. - -transform(Msg = #message{topic = Topic}, #state{topic_prefix = Prefix, topic_suffix = Suffix}) -> - Msg#message{topic = <>}. - diff --git a/src/emqx_local_bridge_sup_sup.erl b/src/emqx_local_bridge_sup_sup.erl deleted file mode 100644 index 8a61d5936..000000000 --- a/src/emqx_local_bridge_sup_sup.erl +++ /dev/null @@ -1,74 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_local_bridge_sup_sup). - --behavior(supervisor). - --include("emqx.hrl"). - --export([start_link/0, bridges/0]). --export([start_bridge/2, start_bridge/3, stop_bridge/2]). - -%% Supervisor callbacks --export([init/1]). - --define(CHILD_ID(Node, Topic), {bridge_sup, Node, Topic}). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc List all bridges --spec(bridges() -> [{node(), emqx_topic:topic(), pid()}]). -bridges() -> - [{Node, Topic, Pid} || {?CHILD_ID(Node, Topic), Pid, supervisor, _} - <- supervisor:which_children(?MODULE)]. - -%% @doc Start a bridge --spec(start_bridge(node(), emqx_topic:topic()) -> {ok, pid()} | {error, term()}). -start_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> - start_bridge(Node, Topic, []). - --spec(start_bridge(node(), emqx_topic:topic(), [emqx_bridge:option()]) - -> {ok, pid()} | {error, term()}). -start_bridge(Node, _Topic, _Options) when Node =:= node() -> - {error, bridge_to_self}; -start_bridge(Node, Topic, Options) when is_atom(Node), is_binary(Topic) -> - Options1 = emqx_misc:merge_opts(emqx_config:get_env(bridge, []), Options), - supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)). - -%% @doc Stop a bridge --spec(stop_bridge(node(), emqx_topic:topic()) -> ok | {error, term()}). -stop_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> - ChildId = ?CHILD_ID(Node, Topic), - case supervisor:terminate_child(?MODULE, ChildId) of - ok -> supervisor:delete_child(?MODULE, ChildId); - Error -> Error - end. - -%%------------------------------------------------------------------------------ -%% Supervisor callbacks -%%------------------------------------------------------------------------------ - -init([]) -> - {ok, {{one_for_one, 10, 3600}, []}}. - -bridge_spec(Node, Topic, Options) -> - #{id => ?CHILD_ID(Node, Topic), - start => {emqx_local_bridge_sup, start_link, [Node, Topic, Options]}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [emqx_local_bridge_sup]}. - diff --git a/src/emqx_logger_formatter.erl b/src/emqx_logger_formatter.erl index 034eb8d36..dd94cceb6 100644 --- a/src/emqx_logger_formatter.erl +++ b/src/emqx_logger_formatter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017-2013-2019. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_logger_handler.erl b/src/emqx_logger_handler.erl new file mode 100644 index 000000000..5a8dc08d1 --- /dev/null +++ b/src/emqx_logger_handler.erl @@ -0,0 +1,42 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_logger_handler). + +-export([log/2]). +-export([init/0]). + +init() -> + logger:add_handler(emqx_logger_handler, + emqx_logger_handler, + #{level => error, + filters => [{easy_filter, {fun filter_by_level/2, []}}], + filters_default => stop}). + +-spec log(LogEvent, Config) -> ok when LogEvent :: logger:log_event(), Config :: logger:handler_config(). +log(#{msg := {report, #{report := [{supervisor, SupName}, + {errorContext, Error}, + {reason, Reason}, + {offender, _}]}}}, _Config) -> + alarm_handler:set_alarm({supervisor_report, [{supervisor, SupName}, + {errorContext, Error}, + {reason, Reason}]}), + ok; +log(_LogEvent, _Config) -> + ok. + +filter_by_level(LogEvent = #{level := error}, _Extra) -> + LogEvent; +filter_by_level(_LogEvent, _Extra) -> + stop. diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl index 20a08ba9f..016ff007f 100644 --- a/src/emqx_mqueue.erl +++ b/src/emqx_mqueue.erl @@ -67,7 +67,7 @@ default_priority => highest | lowest, store_qos0 => boolean() }). --type(message() :: pemqx_types:message()). +-type(message() :: emqx_types:message()). -type(stat() :: {len, non_neg_integer()} | {max_len, non_neg_integer()} diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl new file mode 100644 index 000000000..ba2cc35da --- /dev/null +++ b/src/emqx_os_mon.erl @@ -0,0 +1,153 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_os_mon). + +-behaviour(gen_server). + +-include("logger.hrl"). + +-export([start_link/1]). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-export([get_cpu_check_interval/0, + set_cpu_check_interval/1, + get_cpu_high_watermark/0, + set_cpu_high_watermark/1, + get_cpu_low_watermark/0, + set_cpu_low_watermark/1, + get_mem_check_interval/0, + set_mem_check_interval/1, + get_sysmem_high_watermark/0, + set_sysmem_high_watermark/1, + get_procmem_high_watermark/0, + set_procmem_high_watermark/1]). + +-define(OS_MON, ?MODULE). + +%%---------------------------------------------------------------------- +%% API +%%---------------------------------------------------------------------- + +start_link(Opts) -> + gen_server:start_link({local, ?OS_MON}, ?MODULE, [Opts], []). + +get_cpu_check_interval() -> + call(get_cpu_check_interval). + +set_cpu_check_interval(Seconds) -> + call({set_cpu_check_interval, Seconds}). + +get_cpu_high_watermark() -> + call(get_cpu_high_watermark). + +set_cpu_high_watermark(Float) -> + call({set_cpu_high_watermark, Float}). + +get_cpu_low_watermark() -> + call(get_cpu_low_watermark). + +set_cpu_low_watermark(Float) -> + call({set_cpu_low_watermark, Float}). + +get_mem_check_interval() -> + memsup:get_check_interval() div 1000. + +set_mem_check_interval(Seconds) -> + memsup:set_check_interval(Seconds div 60). + +get_sysmem_high_watermark() -> + memsup:get_sysmem_high_watermark() / 100. + +set_sysmem_high_watermark(Float) -> + memsup:set_sysmem_high_watermark(Float). + +get_procmem_high_watermark() -> + memsup:get_procmem_high_watermark() / 100. + +set_procmem_high_watermark(Float) -> + memsup:set_procmem_high_watermark(Float). + +%%---------------------------------------------------------------------- +%% gen_server callbacks +%%---------------------------------------------------------------------- + +init([Opts]) -> + _ = cpu_sup:util(), + set_mem_check_interval(proplists:get_value(mem_check_interval, Opts, 60)), + set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts, 0.70)), + set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts, 0.05)), + {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts, 0.80), + cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts, 0.60), + cpu_check_interval => proplists:get_value(cpu_check_interval, Opts, 60), + timer => undefined})}. + +handle_call(get_cpu_check_interval, _From, State) -> + {reply, maps:get(cpu_check_interval, State, undefined), State}; +handle_call({set_cpu_check_interval, Seconds}, _From, State) -> + {reply, ok, State#{cpu_check_interval := Seconds}}; + +handle_call(get_cpu_high_watermark, _From, State) -> + {reply, maps:get(cpu_high_watermark, State, undefined), State}; +handle_call({set_cpu_high_watermark, Float}, _From, State) -> + {reply, ok, State#{cpu_high_watermark := Float}}; + +handle_call(get_cpu_low_watermark, _From, State) -> + {reply, maps:get(cpu_low_watermark, State, undefined), State}; +handle_call({set_cpu_low_watermark, Float}, _From, State) -> + {reply, ok, State#{cpu_low_watermark := Float}}; + +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({timeout, Timer, check}, State = #{timer := Timer, + cpu_high_watermark := CPUHighWatermark, + cpu_low_watermark := CPULowWatermark}) -> + case cpu_sup:util() of + 0 -> + {noreply, State#{timer := undefined}}; + {error, Reason} -> + ?LOG(warning, "Failed to get cpu utilization: ~p", [Reason]), + {noreply, ensure_check_timer(State)}; + Busy when Busy / 100 >= CPUHighWatermark -> + alarm_handler:set_alarm({cpu_high_watermark, Busy}), + {noreply, ensure_check_timer(State)}; + Busy when Busy / 100 < CPULowWatermark -> + alarm_handler:clear_alarm(cpu_high_watermark), + {noreply, ensure_check_timer(State)} + end. + +terminate(_Reason, #{timer := Timer}) -> + emqx_misc:cancel_timer(Timer). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%---------------------------------------------------------------------- +%% Internal functions +%%---------------------------------------------------------------------- +call(Req) -> + gen_server:call(?OS_MON, Req, infinity). + +ensure_check_timer(State = #{cpu_check_interval := Interval}) -> + State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}. \ No newline at end of file diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index db1fbc57c..9618a351e 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -29,10 +29,10 @@ -export([parser/1]). -export([session/1]). -export([received/2]). --export([process_packet/2]). +-export([process/2]). -export([deliver/2]). -export([send/2]). --export([shutdown/2]). +-export([terminate/2]). -export_type([state/0]). @@ -53,6 +53,8 @@ clean_start, topic_aliases, packet_size, + will_topic, + will_msg, keepalive, mountpoint, is_super, @@ -65,7 +67,8 @@ connected, connected_at, ignore_loop, - topic_alias_maximum + topic_alias_maximum, + conn_mod }). -opaque(state() :: #pstate{}). @@ -82,7 +85,7 @@ %%------------------------------------------------------------------------------ -spec(init(map(), list()) -> state()). -init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) -> +init(SocketOpts = #{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) -> Zone = proplists:get_value(zone, Options), #pstate{zone = Zone, sendfun = SendFun, @@ -107,7 +110,8 @@ init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) send_stats = #{msg => 0, pkt => 0}, connected = false, ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false), - topic_alias_maximum = #{to_client => 0, from_client => 0}}. + topic_alias_maximum = #{to_client => 0, from_client => 0}, + conn_mod = maps:get(conn_mod, SocketOpts, undefined)}. init_username(Peercert, Options) -> case proplists:get_value(peer_cert_as_username, Options) of @@ -130,11 +134,13 @@ info(PState = #pstate{conn_props = ConnProps, ack_props = AckProps, session = Session, topic_aliases = Aliases, + will_msg = WillMsg, enable_acl = EnableAcl}) -> attrs(PState) ++ [{conn_props, ConnProps}, {ack_props, AckProps}, {session, Session}, {topic_aliases, Aliases}, + {will_msg, WillMsg}, {enable_acl, EnableAcl}]. attrs(#pstate{zone = Zone, @@ -149,7 +155,8 @@ attrs(#pstate{zone = Zone, mountpoint = Mountpoint, is_super = IsSuper, is_bridge = IsBridge, - connected_at = ConnectedAt}) -> + connected_at = ConnectedAt, + conn_mod = ConnMod}) -> [{zone, Zone}, {client_id, ClientId}, {username, Username}, @@ -162,7 +169,8 @@ attrs(#pstate{zone = Zone, {mountpoint, Mountpoint}, {is_super, IsSuper}, {is_bridge, IsBridge}, - {connected_at, ConnectedAt}]. + {connected_at, ConnectedAt}, + {conn_mod, ConnMod}]. attr(max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}) -> get_property('Receive-Maximum', ConnProps, 65535); @@ -218,15 +226,16 @@ parser(#pstate{packet_size = Size, proto_ver = Ver}) -> %% Packet Received %%------------------------------------------------------------------------------ -set_protover(?CONNECT_PACKET(#mqtt_packet_connect{ - proto_ver = ProtoVer}), - PState) -> - PState#pstate{ proto_ver = ProtoVer }; +set_protover(?CONNECT_PACKET(#mqtt_packet_connect{proto_ver = ProtoVer}), PState) -> + PState#pstate{proto_ver = ProtoVer}; set_protover(_Packet, PState) -> PState. --spec(received(emqx_mqtt_types:packet(), state()) -> - {ok, state()} | {error, term()} | {error, term(), state()} | {stop, term(), state()}). +-spec(received(emqx_mqtt_types:packet(), state()) + -> {ok, state()} + | {error, term()} + | {error, term(), state()} + | {stop, term(), state()}). received(?PACKET(Type), PState = #pstate{connected = false}) when Type =/= ?CONNECT -> {error, proto_not_connected, PState}; @@ -234,15 +243,15 @@ received(?PACKET(?CONNECT), PState = #pstate{connected = true}) -> {error, proto_unexpected_connect, PState}; received(Packet = ?PACKET(Type), PState) -> - PState1 = set_protover(Packet, PState), trace(recv, Packet), + PState1 = set_protover(Packet, PState), try emqx_packet:validate(Packet) of true -> case preprocess_properties(Packet, PState1) of + {ok, Packet1, PState2} -> + process(Packet1, inc_stats(recv, Type, PState2)); {error, ReasonCode} -> - {error, ReasonCode, PState1}; - {Packet1, PState2} -> - process_packet(Packet1, inc_stats(recv, Type, PState2)) + {error, ReasonCode, PState1} end catch error:protocol_error -> @@ -268,13 +277,14 @@ received(Packet = ?PACKET(Type), PState) -> %%------------------------------------------------------------------------------ %% Preprocess MQTT Properties %%------------------------------------------------------------------------------ + preprocess_properties(Packet = #mqtt_packet{ variable = #mqtt_packet_connect{ properties = #{'Topic-Alias-Maximum' := ToClient} } }, PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) -> - {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}}; + {ok, Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}}; %% Subscription Identifier preprocess_properties(Packet = #mqtt_packet{ @@ -285,7 +295,7 @@ preprocess_properties(Packet = #mqtt_packet{ }, PState = #pstate{proto_ver = ?MQTT_PROTO_V5}) -> TopicFilters1 = [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters], - {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState}; + {ok, Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState}; %% Topic Alias Mapping preprocess_properties(#mqtt_packet{ @@ -306,8 +316,8 @@ preprocess_properties(Packet = #mqtt_packet{ topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> case AliasId =< TopicAliasMaximum of true -> - {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ - topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + {ok, Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ + topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; false -> deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), {error, ?RC_TOPIC_ALIAS_INVALID} @@ -323,28 +333,28 @@ preprocess_properties(Packet = #mqtt_packet{ topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> case AliasId =< TopicAliasMaximum of true -> - {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + {ok, Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; false -> deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), {error, ?RC_TOPIC_ALIAS_INVALID} end; preprocess_properties(Packet, PState) -> - {Packet, PState}. + {ok, Packet, PState}. %%------------------------------------------------------------------------------ %% Process MQTT Packet %%------------------------------------------------------------------------------ -process_packet(?CONNECT_PACKET( - #mqtt_packet_connect{proto_name = ProtoName, - proto_ver = ProtoVer, - is_bridge = IsBridge, - clean_start = CleanStart, - keepalive = Keepalive, - properties = ConnProps, - client_id = ClientId, - username = Username, - password = Password} = ConnPkt), PState) -> +process(?CONNECT_PACKET( + #mqtt_packet_connect{proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = IsBridge, + clean_start = CleanStart, + keepalive = Keepalive, + properties = ConnProps, + client_id = ClientId, + username = Username, + password = Password} = ConnPkt), PState) -> NewClientId = maybe_use_username_as_clientid(ClientId, Username, PState), @@ -394,17 +404,17 @@ process_packet(?CONNECT_PACKET( {ReasonCode, PState1} end); -process_packet(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PState) -> +process(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PState) -> case check_publish(Packet, PState) of {ok, PState1} -> do_publish(Packet, PState1); {error, ReasonCode} -> ?LOG(warning, "Cannot publish qos0 message to ~s for ~s", - [Topic, emqx_reason_codes:text(ReasonCode)]), + [Topic, emqx_reason_codes:text(ReasonCode)]), do_acl_deny_action(Packet, ReasonCode, PState) end; -process_packet(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PState) -> +process(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PState) -> case check_publish(Packet, PState) of {ok, PState1} -> do_publish(Packet, PState1); @@ -414,30 +424,28 @@ process_packet(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PSta case deliver({puback, PacketId, ReasonCode}, PState) of {ok, PState1} -> do_acl_deny_action(Packet, ReasonCode, PState1); - Error -> - Error + Error -> Error end end; -process_packet(Packet = ?PUBLISH_PACKET(?QOS_2, Topic, PacketId, _Payload), PState) -> +process(Packet = ?PUBLISH_PACKET(?QOS_2, Topic, PacketId, _Payload), PState) -> case check_publish(Packet, PState) of {ok, PState1} -> do_publish(Packet, PState1); {error, ReasonCode} -> ?LOG(warning, "Cannot publish qos2 message to ~s for ~s", - [Topic, emqx_reason_codes:text(ReasonCode)]), + [Topic, emqx_reason_codes:text(ReasonCode)]), case deliver({pubrec, PacketId, ReasonCode}, PState) of {ok, PState1} -> do_acl_deny_action(Packet, ReasonCode, PState1); - Error -> - Error + Error -> Error end end; -process_packet(?PUBACK_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> +process(?PUBACK_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> {ok = emqx_session:puback(SPid, PacketId, ReasonCode), PState}; -process_packet(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> +process(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> case emqx_session:pubrec(SPid, PacketId, ReasonCode) of ok -> send(?PUBREL_PACKET(PacketId), PState); @@ -445,7 +453,7 @@ process_packet(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = send(?PUBREL_PACKET(PacketId, NotFound), PState) end; -process_packet(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> +process(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> case emqx_session:pubrel(SPid, PacketId, ReasonCode) of ok -> send(?PUBCOMP_PACKET(PacketId), PState); @@ -453,22 +461,22 @@ process_packet(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = send(?PUBCOMP_PACKET(PacketId, NotFound), PState) end; -process_packet(?PUBCOMP_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> +process(?PUBCOMP_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> {ok = emqx_session:pubcomp(SPid, PacketId, ReasonCode), PState}; -process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), - PState = #pstate{session = SPid, mountpoint = Mountpoint, - proto_ver = ProtoVer, is_bridge = IsBridge, - ignore_loop = IgnoreLoop}) -> - RawTopicFilters1 = if ProtoVer < ?MQTT_PROTO_V5 -> - IfIgnoreLoop = case IgnoreLoop of true -> 1; false -> 0 end, - case IsBridge of - true -> [{RawTopic, SubOpts#{rap => 1, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters]; - false -> [{RawTopic, SubOpts#{rap => 0, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters] - end; - true -> - RawTopicFilters - end, +process(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), + PState = #pstate{session = SPid, mountpoint = Mountpoint, + proto_ver = ProtoVer, is_bridge = IsBridge, + ignore_loop = IgnoreLoop}) -> + RawTopicFilters1 = if ProtoVer < ?MQTT_PROTO_V5 -> + IfIgnoreLoop = case IgnoreLoop of true -> 1; false -> 0 end, + case IsBridge of + true -> [{RawTopic, SubOpts#{rap => 1, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters]; + false -> [{RawTopic, SubOpts#{rap => 0, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters] + end; + true -> + RawTopicFilters + end, case check_subscribe( parse_topic_filters(?SUBSCRIBE, RawTopicFilters1), PState) of {ok, TopicFilters} -> @@ -483,15 +491,14 @@ process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters) deliver({suback, PacketId, ReasonCodes}, PState) end; {error, TopicFilters} -> - {ReverseSubTopics, ReverseReasonCodes} = - lists:foldl(fun({Topic, #{rc := ?RC_SUCCESS}}, {Topics, Codes}) -> + {SubTopics, ReasonCodes} = + lists:foldr(fun({Topic, #{rc := ?RC_SUCCESS}}, {Topics, Codes}) -> {[Topic|Topics], [?RC_IMPLEMENTATION_SPECIFIC_ERROR | Codes]}; ({Topic, #{rc := Code}}, {Topics, Codes}) -> {[Topic|Topics], [Code|Codes]} end, {[], []}, TopicFilters), - {SubTopics, ReasonCodes} = {lists:reverse(ReverseSubTopics), lists:reverse(ReverseReasonCodes)}, ?LOG(warning, "Cannot subscribe ~p for ~p", - [SubTopics, [emqx_reason_codes:text(R) || R <- ReasonCodes]]), + [SubTopics, [emqx_reason_codes:text(R) || R <- ReasonCodes]]), case deliver({suback, PacketId, ReasonCodes}, PState) of {ok, PState1} -> do_acl_deny_action(Packet, ReasonCodes, PState1); @@ -500,8 +507,8 @@ process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters) end end; -process_packet(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), - PState = #pstate{session = SPid, mountpoint = MountPoint}) -> +process(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), + PState = #pstate{session = SPid, mountpoint = MountPoint}) -> case emqx_hooks:run('client.unsubscribe', [credentials(PState)], parse_topic_filters(?UNSUBSCRIBE, RawTopicFilters)) of {ok, TopicFilters} -> @@ -514,22 +521,25 @@ process_packet(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), deliver({unsuback, PacketId, ReasonCodes}, PState) end; -process_packet(?PACKET(?PINGREQ), PState) -> +process(?PACKET(?PINGREQ), PState) -> send(?PACKET(?PINGRESP), PState); -process_packet(?DISCONNECT_PACKET(?RC_SUCCESS, #{'Session-Expiry-Interval' := Interval}), - PState = #pstate{session = SPid, conn_props = #{'Session-Expiry-Interval' := OldInterval}}) -> +process(?DISCONNECT_PACKET(?RC_SUCCESS, #{'Session-Expiry-Interval' := Interval}), + PState = #pstate{session = SPid, conn_props = #{'Session-Expiry-Interval' := OldInterval}}) -> case Interval =/= 0 andalso OldInterval =:= 0 of true -> deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState), - {error, protocol_error, PState}; + {error, protocol_error, PState#pstate{will_msg = undefined}}; false -> emqx_session:update_expiry_interval(SPid, Interval), - {stop, normal, PState} + %% Clean willmsg + {stop, normal, PState#pstate{will_msg = undefined}} end; -process_packet(?DISCONNECT_PACKET(?RC_SUCCESS), PState) -> - {stop, normal, PState}; -process_packet(?DISCONNECT_PACKET(_), PState) -> + +process(?DISCONNECT_PACKET(?RC_SUCCESS), PState) -> + {stop, normal, PState#pstate{will_msg = undefined}}; + +process(?DISCONNECT_PACKET(_), PState) -> {stop, {shutdown, abnormal_disconnet}, PState}. %%------------------------------------------------------------------------------ @@ -562,15 +572,16 @@ do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId), puback(?QOS_0, _PacketId, _Result, PState) -> {ok, PState}; -puback(?QOS_1, PacketId, [], PState) -> +puback(?QOS_1, PacketId, {ok, []}, PState) -> deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch? +%%TODO: calc the deliver count? +puback(?QOS_1, PacketId, {ok, _Result}, PState) -> deliver({puback, PacketId, ?RC_SUCCESS}, PState); puback(?QOS_1, PacketId, {error, ReasonCode}, PState) -> deliver({puback, PacketId, ReasonCode}, PState); -puback(?QOS_2, PacketId, [], PState) -> +puback(?QOS_2, PacketId, {ok, []}, PState) -> deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); -puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch? +puback(?QOS_2, PacketId, {ok, _Result}, PState) -> deliver({pubrec, PacketId, ?RC_SUCCESS}, PState); puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> deliver({pubrec, PacketId, ReasonCode}, PState). @@ -579,7 +590,17 @@ puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> %% Deliver Packet -> Client %%------------------------------------------------------------------------------ --spec(deliver(tuple(), state()) -> {ok, state()} | {error, term()}). +-spec(deliver(list(tuple()) | tuple(), state()) -> {ok, state()} | {error, term()}). +deliver([], PState) -> + {ok, PState}; +deliver([Pub|More], PState) -> + case deliver(Pub, PState) of + {ok, PState1} -> + deliver(More, PState1); + {error, _} = Error -> + Error + end; + deliver({connack, ReasonCode}, PState) -> send(?CONNACK_PACKET(ReasonCode), PState); @@ -666,11 +687,13 @@ deliver({disconnect, _ReasonCode}, PState) -> %% Send Packet to Client -spec(send(emqx_mqtt_types:packet(), state()) -> {ok, state()} | {error, term()}). -send(Packet = ?PACKET(Type), PState = #pstate{proto_ver = Ver, sendfun = SendFun}) -> - trace(send, Packet), - case SendFun(Packet, #{version => Ver}) of +send(Packet = ?PACKET(Type), PState = #pstate{proto_ver = Ver, sendfun = Send}) -> + Data = emqx_frame:serialize(Packet, #{version => Ver}), + case Send(Data) of ok -> + trace(send, Packet), emqx_metrics:sent(Packet), + emqx_metrics:trans(inc, 'bytes/sent', iolist_size(Data)), {ok, inc_stats(send, Type, PState)}; {error, Reason} -> {error, Reason} @@ -809,14 +832,13 @@ check_will_topic(#mqtt_packet_connect{will_topic = WillTopic} = ConnPkt, PState) {error, ?RC_TOPIC_NAME_INVALID} end. -check_will_acl(_ConnPkt, #pstate{enable_acl = EnableAcl}) - when not EnableAcl -> +check_will_acl(_ConnPkt, #pstate{enable_acl = EnableAcl}) when not EnableAcl -> ok; check_will_acl(#mqtt_packet_connect{will_topic = WillTopic}, PState) -> case emqx_access_control:check_acl(credentials(PState), publish, WillTopic) of allow -> ok; deny -> - ?LOG(warning, "Will message (to ~s) validation failed, acl denied", [WillTopic]), + ?LOG(warning, "Cannot publish will message to ~p for acl denied", [WillTopic]), {error, ?RC_NOT_AUTHORIZED} end. @@ -825,7 +847,7 @@ check_publish(Packet, PState) -> fun check_pub_acl/2], Packet, PState). check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ properties = _Properties}}, + variable = #mqtt_packet_publish{properties = _Properties}}, #pstate{zone = Zone}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}). @@ -892,15 +914,15 @@ inc_stats(Type, Stats = #{pkt := PktCnt, msg := MsgCnt}) -> false -> MsgCnt end}. -shutdown(_Reason, #pstate{client_id = undefined}) -> +terminate(_Reason, #pstate{client_id = undefined}) -> ok; -shutdown(_Reason, #pstate{connected = false}) -> +terminate(_Reason, #pstate{connected = false}) -> ok; -shutdown(conflict, _PState) -> +terminate(conflict, _PState) -> ok; -shutdown(discard, _PState) -> +terminate(discard, _PState) -> ok; -shutdown(Reason, PState) -> +terminate(Reason, PState) -> ?LOG(info, "Shutdown for ~p", [Reason]), emqx_hooks:run('client.disconnected', [credentials(PState), Reason]). diff --git a/src/emqx_rpc.erl b/src/emqx_rpc.erl index 0245da838..e0d82f400 100644 --- a/src/emqx_rpc.erl +++ b/src/emqx_rpc.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,4 +28,3 @@ multicall(Nodes, Mod, Fun, Args) -> cast(Node, Mod, Fun, Args) -> ?RPC:cast(Node, Mod, Fun, Args). - diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 06d026e8c..276454cd9 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -71,8 +71,11 @@ %% Clean Start Flag clean_start = false :: boolean(), - %% Client Binding: local | remote - binding = local :: local | remote, + %% Conn Binding: local | remote + %% binding = local :: local | remote, + + %% Deliver fun + deliver_fun :: function(), %% ClientId: Identifier of Session client_id :: binary(), @@ -157,6 +160,8 @@ -export_type([attr/0]). +-define(DEFAULT_BATCH_N, 1000). + %% @doc Start a session proc. -spec(start_link(SessAttrs :: map()) -> {ok, pid()}). start_link(SessAttrs) -> @@ -196,13 +201,13 @@ attrs(SPid) when is_pid(SPid) -> gen_server:call(SPid, attrs, infinity); attrs(#state{clean_start = CleanStart, - binding = Binding, client_id = ClientId, + conn_pid = ConnPid, username = Username, expiry_interval = ExpiryInterval, created_at = CreatedAt}) -> [{clean_start, CleanStart}, - {binding, Binding}, + {binding, binding(ConnPid)}, {client_id, ClientId}, {username, Username}, {expiry_interval, ExpiryInterval div 1000}, @@ -249,19 +254,19 @@ subscribe(SPid, PacketId, Properties, TopicFilters) -> %% @doc Called by connection processes when publishing messages -spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message()) - -> emqx_types:deliver_results() | {error, term()}). + -> {ok, emqx_types:deliver_results()} | {error, term()}). publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) -> %% Publish QoS0 message directly - emqx_broker:publish(Msg); + {ok, emqx_broker:publish(Msg)}; publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) -> %% Publish QoS1 message directly - emqx_broker:publish(Msg); + {ok, emqx_broker:publish(Msg)}; publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> %% Register QoS2 message packet ID (and timestamp) to session, then publish case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of - ok -> emqx_broker:publish(Msg); + ok -> {ok, emqx_broker:publish(Msg)}; {error, Reason} -> {error, Reason} end. @@ -342,7 +347,7 @@ init([Parent, #{zone := Zone, IdleTimout = get_env(Zone, idle_timeout, 30000), State = #state{idle_timeout = IdleTimout, clean_start = CleanStart, - binding = binding(ConnPid), + deliver_fun = deliver_fun(ConnPid), client_id = ClientId, username = Username, conn_pid = ConnPid, @@ -376,9 +381,18 @@ init_mqueue(Zone) -> default_priority => get_env(Zone, mqueue_default_priority) }). +binding(undefined) -> undefined; binding(ConnPid) -> case node(ConnPid) =:= node() of true -> local; false -> remote end. +deliver_fun(ConnPid) when node(ConnPid) == node() -> + fun(Packet) -> ConnPid ! {deliver, Packet}, ok end; +deliver_fun(ConnPid) -> + Node = node(ConnPid), + fun(Packet) -> + emqx_rpc:cast(Node, erlang, send, [ConnPid, {deliver, Packet}]) + end. + handle_call(info, _From, State) -> reply(info(State), State); @@ -539,7 +553,7 @@ handle_cast({resume, #{conn_pid := ConnPid, true = link(ConnPid), State1 = State#state{conn_pid = ConnPid, - binding = binding(ConnPid), + deliver_fun = deliver_fun(ConnPid), old_conn_pid = OldConnPid, clean_start = false, retry_timer = undefined, @@ -566,25 +580,11 @@ handle_cast(Msg, State) -> emqx_logger:error("[Session] unexpected cast: ~p", [Msg]), {noreply, State}. -%% Batch dispatch -handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) -> - noreply(lists:foldl( - fun(Msg, St) -> - element(2, handle_info({dispatch, Topic, Msg}, St)) - end, State, Msgs)); +handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, message) -> + handle_dispatch([{Topic, Msg}], State); -%% Dispatch message -handle_info({dispatch, Topic, Msg = #message{}}, State) -> - case emqx_shared_sub:is_ack_required(Msg) andalso not has_connection(State) of - true -> - %% Require ack, but we do not have connection - %% negative ack the message so it can try the next subscriber in the group - ok = emqx_shared_sub:nack_no_connection(Msg), - {noreply, State}; - false -> - NewState = handle_dispatch(Topic, Msg, State), - noreply(ensure_stats_timer(maybe_gc({1, msg_size(Msg)}, NewState))) - end; +handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) -> + handle_dispatch([{Topic, Msg} || Msg <- Msgs], State); %% Do nothing if the client has been disconnected. handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) -> @@ -684,18 +684,11 @@ maybe_shutdown(Pid, Reason) -> %% Internal functions %%------------------------------------------------------------------------------ -has_connection(#state{conn_pid = Pid}) -> +is_connection_alive(#state{conn_pid = Pid}) -> is_pid(Pid) andalso is_process_alive(Pid). -handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) -> - case maps:find(Topic, SubMap) of - {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); - {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> - run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); - error -> - dispatch(emqx_message:unset_flag(dup, Msg), State) - end. +%%------------------------------------------------------------------------------ +%% Suback and unsuback suback(_From, undefined, _ReasonCodes) -> ignore; @@ -722,7 +715,6 @@ kick(ClientId, OldConnPid, ConnPid) -> %%------------------------------------------------------------------------------ %% Replay or Retry Delivery -%%------------------------------------------------------------------------------ %% Redeliver at once if force is true retry_delivery(Force, State = #state{inflight = Inflight}) -> @@ -766,6 +758,7 @@ retry_delivery(Force, [{Type, Msg0, Ts} | Msgs], Now, %%------------------------------------------------------------------------------ %% Send Will Message %%------------------------------------------------------------------------------ + send_willmsg(undefined) -> ignore; send_willmsg(WillMsg) -> @@ -801,64 +794,156 @@ expire_awaiting_rel([{PacketId, Ts} | More], Now, is_awaiting_full(#state{max_awaiting_rel = 0}) -> false; -is_awaiting_full(#state{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLen}) -> +is_awaiting_full(#state{awaiting_rel = AwaitingRel, + max_awaiting_rel = MaxLen}) -> maps:size(AwaitingRel) >= MaxLen. %%------------------------------------------------------------------------------ -%% Dispatch Messages +%% Dispatch messages %%------------------------------------------------------------------------------ -run_dispatch_steps([], Msg, State) -> - dispatch(Msg, State); -run_dispatch_steps([{nl, 1}|_Steps], #message{from = ClientId}, State = #state{client_id = ClientId}) -> - State; -run_dispatch_steps([{nl, _}|Steps], Msg, State) -> - run_dispatch_steps(Steps, Msg, State); -run_dispatch_steps([{qos, SubQoS}|Steps], Msg0 = #message{qos = PubQoS}, State = #state{upgrade_qos = false}) -> - %% Ack immediately if a shared dispatch QoS is downgraded to 0 - Msg = case SubQoS =:= ?QOS_0 of - true -> emqx_shared_sub:maybe_ack(Msg0); - false -> Msg0 - end, - run_dispatch_steps(Steps, Msg#message{qos = min(SubQoS, PubQoS)}, State); -run_dispatch_steps([{qos, SubQoS}|Steps], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) -> - run_dispatch_steps(Steps, Msg#message{qos = max(SubQoS, PubQoS)}, State); -run_dispatch_steps([{rap, _Rap}|Steps], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) -> - run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, true, Flags)}, State); -run_dispatch_steps([{rap, 0}|Steps], Msg = #message{flags = Flags}, State = #state{}) -> - run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, false, Flags)}, State); -run_dispatch_steps([{rap, _}|Steps], Msg, State) -> - run_dispatch_steps(Steps, Msg, State); -run_dispatch_steps([{subid, SubId}|Steps], Msg, State) -> - run_dispatch_steps(Steps, emqx_message:set_header('Subscription-Identifier', SubId, Msg), State). +handle_dispatch(Msgs, State = #state{inflight = Inflight, subscriptions = SubMap}) -> + %% Drain the mailbox and batch deliver + Msgs1 = drain_m(batch_n(Inflight), Msgs), + %% Ack the messages for shared subscription + Msgs2 = maybe_ack_shared(Msgs1, State), + %% Process suboptions + Msgs3 = lists:foldr( + fun({Topic, Msg}, Acc) -> + SubOpts = find_subopts(Topic, SubMap), + case process_subopts(SubOpts, Msg, State) of + {ok, Msg1} -> [Msg1|Acc]; + ignore -> Acc + end + end, [], Msgs2), + NState = batch_process(Msgs3, State), + noreply(ensure_stats_timer(NState)). + +batch_n(Inflight) -> + case emqx_inflight:max_size(Inflight) of + 0 -> ?DEFAULT_BATCH_N; + Sz -> Sz - emqx_inflight:size(Inflight) + end. + +drain_m(Cnt, Msgs) when Cnt =< 0 -> + lists:reverse(Msgs); +drain_m(Cnt, Msgs) -> + receive + {dispatch, Topic, Msg} -> + drain_m(Cnt-1, [{Topic, Msg}|Msgs]) + after 0 -> + lists:reverse(Msgs) + end. + +%% Ack or nack the messages of shared subscription? +maybe_ack_shared(Msgs, State) when is_list(Msgs) -> + lists:foldr( + fun({Topic, Msg}, Acc) -> + case maybe_ack_shared(Msg, State) of + ok -> Acc; + Msg1 -> [{Topic, Msg1}|Acc] + end + end, [], Msgs); + +maybe_ack_shared(Msg, State) -> + case emqx_shared_sub:is_ack_required(Msg) of + true -> do_ack_shared(Msg, State); + false -> Msg + end. + +do_ack_shared(Msg, State = #state{inflight = Inflight}) -> + case {is_connection_alive(State), + emqx_inflight:is_full(Inflight)} of + {false, _} -> + %% Require ack, but we do not have connection + %% negative ack the message so it can try the next subscriber in the group + emqx_shared_sub:nack_no_connection(Msg); + {_, true} -> + emqx_shared_sub:maybe_nack_dropped(Msg); + _ -> + %% Ack QoS1/QoS2 messages when message is delivered to connection. + %% NOTE: NOT to wait for PUBACK because: + %% The sender is monitoring this session process, + %% if the message is delivered to client but connection or session crashes, + %% sender will try to dispatch the message to the next shared subscriber. + %% This violates spec as QoS2 messages are not allowed to be sent to more + %% than one member in the group. + emqx_shared_sub:maybe_ack(Msg) + end. + +process_subopts([], Msg, _State) -> + {ok, Msg}; +process_subopts([{nl, 1}|_Opts], #message{from = ClientId}, #state{client_id = ClientId}) -> + ignore; +process_subopts([{nl, _}|Opts], Msg, State) -> + process_subopts(Opts, Msg, State); +process_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = false}) -> + process_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, State); +process_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) -> + process_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, State); +process_subopts([{rap, _Rap}|Opts], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) -> + process_subopts(Opts, Msg#message{flags = maps:put(retain, true, Flags)}, State); +process_subopts([{rap, 0}|Opts], Msg = #message{flags = Flags}, State = #state{}) -> + process_subopts(Opts, Msg#message{flags = maps:put(retain, false, Flags)}, State); +process_subopts([{rap, _}|Opts], Msg, State) -> + process_subopts(Opts, Msg, State); +process_subopts([{subid, SubId}|Opts], Msg, State) -> + process_subopts(Opts, emqx_message:set_header('Subscription-Identifier', SubId, Msg), State). + +find_subopts(Topic, SubMap) -> + case maps:find(Topic, SubMap) of + {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> + [{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}]; + {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> + [{nl, Nl}, {qos, QoS}, {rap, Rap}]; + error -> [] + end. + +batch_process(Msgs, State) -> + {ok, Publishes, NState} = process_msgs(Msgs, [], State), + ok = batch_deliver(Publishes, NState), + maybe_gc(msg_cnt(Msgs), NState). + +process_msgs([], Publishes, State) -> + {ok, lists:reverse(Publishes), State}; + +process_msgs([Msg|Msgs], Publishes, State) -> + case process_msg(Msg, State) of + {ok, Publish, NState} -> + process_msgs(Msgs, [Publish|Publishes], NState); + {ignore, NState} -> + process_msgs(Msgs, Publishes, NState) + end. %% Enqueue message if the client has been disconnected -dispatch(Msg, State = #state{client_id = ClientId, username = Username, conn_pid = undefined}) -> - case emqx_hooks:run('message.dropped', [#{client_id => ClientId, username => Username}, Msg]) of - ok -> enqueue_msg(Msg, State); - stop -> State - end; +process_msg(Msg, State = #state{conn_pid = undefined}) -> + {ignore, enqueue_msg(Msg, State)}; -%% Deliver qos0 message directly to client -dispatch(Msg = #message{qos = ?QOS_0} = Msg, State) -> - ok = deliver(undefined, Msg, State), - State; +%% Prepare the qos0 message delivery +process_msg(Msg = #message{qos = ?QOS_0}, State) -> + {ok, {publish, undefined, Msg}, State}; -dispatch(Msg = #message{qos = QoS} = Msg, - State = #state{next_pkt_id = PacketId, inflight = Inflight}) +process_msg(Msg = #message{qos = QoS}, + State = #state{next_pkt_id = PacketId, inflight = Inflight}) when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> case emqx_inflight:is_full(Inflight) of true -> - enqueue_msg(Msg, State); + {ignore, enqueue_msg(Msg, State)}; false -> - ok = deliver(PacketId, Msg, State), - await(PacketId, Msg, next_pkt_id(State)) + Publish = {publish, PacketId, Msg}, + NState = await(PacketId, Msg, State), + {ok, Publish, next_pkt_id(NState)} end. -enqueue_msg(Msg, State = #state{mqueue = Q}) -> +enqueue_msg(Msg, State = #state{mqueue = Q, client_id = ClientId, username = Username}) -> emqx_pd:update_counter(enqueue_stats, 1), {Dropped, NewQ} = emqx_mqueue:in(Msg, Q), - Dropped =/= undefined andalso emqx_shared_sub:maybe_nack_dropped(Dropped), + if + Dropped =/= undefined -> + SessProps = #{client_id => ClientId, username => Username}, + emqx_hooks:run('message.dropped', [SessProps, Msg]); + true -> ok + end, State#state{mqueue = NewQ}. %%------------------------------------------------------------------------------ @@ -866,28 +951,22 @@ enqueue_msg(Msg, State = #state{mqueue = Q}) -> %%------------------------------------------------------------------------------ redeliver({PacketId, Msg = #message{qos = QoS}}, State) -> - deliver(PacketId, if QoS =:= ?QOS_2 -> Msg; - true -> emqx_message:set_flag(dup, Msg) - end, State); + Msg1 = if + QoS =:= ?QOS_2 -> Msg; + true -> emqx_message:set_flag(dup, Msg) + end, + do_deliver(PacketId, Msg1, State); -redeliver({pubrel, PacketId}, #state{conn_pid = ConnPid}) -> - ConnPid ! {deliver, {pubrel, PacketId}}. +redeliver({pubrel, PacketId}, #state{deliver_fun = DeliverFun}) -> + DeliverFun({pubrel, PacketId}). -deliver(PacketId, Msg, State) -> +do_deliver(PacketId, Msg, #state{deliver_fun = DeliverFun}) -> emqx_pd:update_counter(deliver_stats, 1), - %% Ack QoS1/QoS2 messages when message is delivered to connection. - %% NOTE: NOT to wait for PUBACK because: - %% The sender is monitoring this session process, - %% if the message is delivered to client but connection or session crashes, - %% sender will try to dispatch the message to the next shared subscriber. - %% This violates spec as QoS2 messages are not allowed to be sent to more - %% than one member in the group. - do_deliver(PacketId, emqx_shared_sub:maybe_ack(Msg), State). + DeliverFun({publish, PacketId, Msg}). -do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = local}) -> - ConnPid ! {deliver, {publish, PacketId, Msg}}, ok; -do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = remote}) -> - emqx_rpc:cast(node(ConnPid), erlang, send, [ConnPid, {deliver, {publish, PacketId, Msg}}]). +batch_deliver(Publishes, #state{deliver_fun = DeliverFun}) -> + emqx_pd:update_counter(deliver_stats, length(Publishes)), + DeliverFun(Publishes). %%------------------------------------------------------------------------------ %% Awaiting ACK for QoS1/QoS2 Messages @@ -932,26 +1011,31 @@ acked(pubcomp, PacketId, State = #state{inflight = Inflight}) -> dequeue(State = #state{conn_pid = undefined}) -> State; -dequeue(State = #state{inflight = Inflight}) -> - case emqx_inflight:is_full(Inflight) of - true -> State; - false -> dequeue2(State) +dequeue(State = #state{inflight = Inflight, mqueue = Q}) -> + case emqx_mqueue:is_empty(Q) + orelse emqx_inflight:is_full(Inflight) of + true -> State; + false -> + {Msgs, Q1} = drain_q(batch_n(Inflight), [], Q), + batch_process(lists:reverse(Msgs), State#state{mqueue = Q1}) end. -dequeue2(State = #state{mqueue = Q}) -> +drain_q(Cnt, Msgs, Q) when Cnt =< 0 -> + {Msgs, Q}; + +drain_q(Cnt, Msgs, Q) -> case emqx_mqueue:out(Q) of - {empty, _Q} -> State; + {empty, _Q} -> {Msgs, Q}; {{value, Msg}, Q1} -> - %% Dequeue more - dequeue(dispatch(Msg, State#state{mqueue = Q1})) + drain_q(Cnt-1, [Msg|Msgs], Q1) end. %%------------------------------------------------------------------------------ %% Ensure timers -ensure_await_rel_timer(State = #state{await_rel_timer = undefined, await_rel_timeout = Timeout}) -> +ensure_await_rel_timer(State = #state{await_rel_timer = undefined, + await_rel_timeout = Timeout}) -> ensure_await_rel_timer(Timeout, State); - ensure_await_rel_timer(State) -> State. @@ -960,7 +1044,8 @@ ensure_await_rel_timer(Timeout, State = #state{await_rel_timer = undefined}) -> ensure_await_rel_timer(_Timeout, State) -> State. -ensure_retry_timer(State = #state{retry_timer = undefined, retry_interval = Interval}) -> +ensure_retry_timer(State = #state{retry_timer = undefined, + retry_interval = Interval}) -> ensure_retry_timer(Interval, State); ensure_retry_timer(State) -> State. @@ -970,7 +1055,8 @@ ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) -> ensure_retry_timer(_Timeout, State) -> State. -ensure_expire_timer(State = #state{expiry_interval = Interval}) when Interval > 0 andalso Interval =/= 16#ffffffff -> +ensure_expire_timer(State = #state{expiry_interval = Interval}) + when Interval > 0 andalso Interval =/= 16#ffffffff -> State#state{expiry_timer = emqx_misc:start_timer(Interval * 1000, expired)}; ensure_expire_timer(State) -> State. @@ -997,15 +1083,20 @@ next_pkt_id(State = #state{next_pkt_id = 16#FFFF}) -> next_pkt_id(State = #state{next_pkt_id = Id}) -> State#state{next_pkt_id = Id + 1}. +%%------------------------------------------------------------------------------ +%% Maybe GC + +msg_cnt(Msgs) -> + lists:foldl(fun(Msg, {Cnt, Oct}) -> + {Cnt+1, Oct+msg_size(Msg)} + end, {0, 0}, Msgs). + %% Take only the payload size into account, add other fields if necessary msg_size(#message{payload = Payload}) -> payload_size(Payload). %% Payload should be binary(), but not 100% sure. Need dialyzer! payload_size(Payload) -> erlang:iolist_size(Payload). -%%------------------------------------------------------------------------------ -%% Maybe GC - maybe_gc(_, State = #state{gc_state = undefined}) -> State; maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 5f62df904..eff33a841 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -61,9 +61,6 @@ init([]) -> RouterSup = supervisor_spec(emqx_router_sup), %% Broker Sup BrokerSup = supervisor_spec(emqx_broker_sup), - %% BridgeSup - LocalBridgeSup = supervisor_spec(emqx_local_bridge_sup_sup), - BridgeSup = supervisor_spec(emqx_bridge_sup), %% AccessControl AccessControl = worker_spec(emqx_access_control), @@ -77,7 +74,6 @@ init([]) -> [KernelSup, RouterSup, BrokerSup, - LocalBridgeSup, BridgeSup, AccessControl, SMSup, @@ -92,4 +88,3 @@ worker_spec(M) -> {M, {M, start_link, []}, permanent, 30000, worker, [M]}. supervisor_spec(M) -> {M, {M, start_link, []}, permanent, infinity, supervisor, [M]}. - diff --git a/src/emqx_sys_mon.erl b/src/emqx_sys_mon.erl index 44a256f3f..11a3c8ebe 100644 --- a/src/emqx_sys_mon.erl +++ b/src/emqx_sys_mon.erl @@ -163,5 +163,6 @@ safe_publish(Event, WarnMsg) -> emqx_broker:safe_publish(sysmon_msg(Topic, iolist_to_binary(WarnMsg))). sysmon_msg(Topic, Payload) -> - emqx_message:make(?SYSMON, #{sys => true}, Topic, Payload). + Msg = emqx_message:make(?SYSMON, Topic, Payload), + emqx_message:set_flag(sys, Msg). diff --git a/src/emqx_sys_sup.erl b/src/emqx_sys_sup.erl index 24609acdb..9341b8528 100644 --- a/src/emqx_sys_sup.erl +++ b/src/emqx_sys_sup.erl @@ -24,17 +24,23 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Sys = #{id => sys, - start => {emqx_sys, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_sys]}, - Sysmon = #{id => sys_mon, - start => {emqx_sys_mon, start_link, [emqx_config:get_env(sysmon, [])]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_sys_mon]}, - {ok, {{one_for_one, 10, 100}, [Sys, Sysmon]}}. + {ok, {{one_for_one, 10, 100}, [child_spec(emqx_sys, worker), + child_spec(emqx_sys_mon, worker, [emqx_config:get_env(sysmon, [])]), + child_spec(emqx_os_mon, worker, [emqx_config:get_env(os_mon, [])]), + child_spec(emqx_vm_mon, worker, [emqx_config:get_env(vm_mon, [])])]}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +child_spec(M, worker) -> + child_spec(M, worker, []). + +child_spec(M, worker, A) -> + #{id => M, + start => {M, start_link, A}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [M]}. diff --git a/src/emqx_topic.erl b/src/emqx_topic.erl index 59f592984..bb615ccfe 100644 --- a/src/emqx_topic.erl +++ b/src/emqx_topic.erl @@ -20,7 +20,7 @@ -export([triples/1]). -export([words/1]). -export([wildcard/1]). --export([join/1]). +-export([join/1, prepend/2]). -export([feed_var/3]). -export([systop/1]). -export([parse/1, parse/2]). @@ -129,10 +129,23 @@ join(root, W) -> join(Parent, W) -> <<(bin(Parent))/binary, $/, (bin(W))/binary>>. +%% @doc Prepend a topic prefix. +%% Ensured to have only one / between prefix and suffix. +prepend(root, W) -> bin(W); +prepend(undefined, W) -> bin(W); +prepend(<<>>, W) -> bin(W); +prepend(Parent0, W) -> + Parent = bin(Parent0), + case binary:last(Parent) of + $/ -> <>; + _ -> join(Parent, W) + end. + bin('') -> <<>>; bin('+') -> <<"+">>; bin('#') -> <<"#">>; -bin(B) when is_binary(B) -> B. +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L). levels(Topic) when is_binary(Topic) -> length(words(Topic)). diff --git a/src/emqx_types.erl b/src/emqx_types.erl index d84b1099a..2fe34e853 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -31,7 +31,7 @@ -type(pubsub() :: publish | subscribe). -type(topic() :: binary()). -type(subid() :: binary() | atom()). --type(subopts() :: #{qos := integer(), +-type(subopts() :: #{qos := emqx_mqtt_types:qos(), share => binary(), atom() => term() }). @@ -59,4 +59,3 @@ -type(alarm() :: #alarm{}). -type(plugin() :: #plugin{}). -type(command() :: #command{}). - diff --git a/src/emqx_vm_mon.erl b/src/emqx_vm_mon.erl new file mode 100644 index 000000000..a4562198e --- /dev/null +++ b/src/emqx_vm_mon.erl @@ -0,0 +1,118 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_vm_mon). + +-behaviour(gen_server). + +-export([start_link/1]). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-export([get_check_interval/0, + set_check_interval/1, + get_process_high_watermark/0, + set_process_high_watermark/1, + get_process_low_watermark/0, + set_process_low_watermark/1]). + +-define(VM_MON, ?MODULE). + +%%---------------------------------------------------------------------- +%% API +%%---------------------------------------------------------------------- + +start_link(Opts) -> + gen_server:start_link({local, ?VM_MON}, ?MODULE, [Opts], []). + +get_check_interval() -> + call(get_check_interval). + +set_check_interval(Seconds) -> + call({set_check_interval, Seconds}). + +get_process_high_watermark() -> + call(get_process_high_watermark). + +set_process_high_watermark(Float) -> + call({set_process_high_watermark, Float}). + +get_process_low_watermark() -> + call(get_process_low_watermark). + +set_process_low_watermark(Float) -> + call({set_process_low_watermark, Float}). + +%%---------------------------------------------------------------------- +%% gen_server callbacks +%%---------------------------------------------------------------------- + +init([Opts]) -> + {ok, ensure_check_timer(#{check_interval => proplists:get_value(check_interval, Opts, 30), + process_high_watermark => proplists:get_value(process_high_watermark, Opts, 0.70), + process_low_watermark => proplists:get_value(process_low_watermark, Opts, 0.50), + timer => undefined})}. + +handle_call(get_check_interval, _From, State) -> + {reply, maps:get(check_interval, State, undefined), State}; +handle_call({set_check_interval, Seconds}, _From, State) -> + {reply, ok, State#{check_interval := Seconds}}; + +handle_call(get_process_high_watermark, _From, State) -> + {reply, maps:get(process_high_watermark, State, undefined), State}; +handle_call({set_process_high_watermark, Float}, _From, State) -> + {reply, ok, State#{process_high_watermark := Float}}; + +handle_call(get_process_low_watermark, _From, State) -> + {reply, maps:get(process_low_watermark, State, undefined), State}; +handle_call({set_process_low_watermark, Float}, _From, State) -> + {reply, ok, State#{process_low_watermark := Float}}; + +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({timeout, Timer, check}, State = #{timer := Timer, + process_high_watermark := ProcHighWatermark, + process_low_watermark := ProcLowWatermark}) -> + ProcessCount = erlang:system_info(process_count), + case ProcessCount / erlang:system_info(process_limit) of + Percent when Percent >= ProcHighWatermark -> + alarm_handler:set_alarm({too_many_processes, ProcessCount}); + Percent when Percent < ProcLowWatermark -> + alarm_handler:clear_alarm(too_many_processes) + end, + {noreply, ensure_check_timer(State)}. + +terminate(_Reason, #{timer := Timer}) -> + emqx_misc:cancel_timer(Timer). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%---------------------------------------------------------------------- +%% Internal functions +%%---------------------------------------------------------------------- +call(Req) -> + gen_server:call(?VM_MON, Req, infinity). + +ensure_check_timer(State = #{check_interval := Interval}) -> + State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}. \ No newline at end of file diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index fec52995f..0c88751fc 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -18,7 +18,8 @@ -include("emqx_mqtt.hrl"). -include("logger.hrl"). --export([info/1, attrs/1]). +-export([info/1]). +-export([attrs/1]). -export([stats/1]). -export([kick/1]). -export([session/1]). @@ -37,7 +38,7 @@ sockname, idle_timeout, proto_state, - parser_state, + parse_state, keepalive, enable_stats, stats_timer, @@ -127,25 +128,23 @@ websocket_init(#state{request = Req, options = Options}) -> ProtoState = emqx_protocol:init(#{peername => Peername, sockname => Sockname, peercert => Peercert, - sendfun => send_fun(self())}, Options), + sendfun => send_fun(self()), + conn_mod => ?MODULE}, Options), ParserState = emqx_protocol:parser(ProtoState), Zone = proplists:get_value(zone, Options), EnableStats = emqx_zone:get_env(Zone, enable_stats, true), IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), - emqx_logger:set_metadata_peername(esockd_net:format(Peername)), {ok, #state{peername = Peername, sockname = Sockname, - parser_state = ParserState, + parse_state = ParserState, proto_state = ProtoState, enable_stats = EnableStats, idle_timeout = IdleTimout}}. send_fun(WsPid) -> - fun(Packet, Options) -> - Data = emqx_frame:serialize(Packet, Options), + fun(Data) -> BinSize = iolist_size(Data), - emqx_metrics:trans(inc, 'bytes/sent', BinSize), emqx_pd:update_counter(send_cnt, 1), emqx_pd:update_counter(send_oct, BinSize), WsPid ! {binary, iolist_to_binary(Data)}, @@ -159,15 +158,15 @@ websocket_handle({binary, <<>>}, State) -> {ok, ensure_stats_timer(State)}; websocket_handle({binary, [<<>>]}, State) -> {ok, ensure_stats_timer(State)}; -websocket_handle({binary, Data}, State = #state{parser_state = ParserState, - proto_state = ProtoState}) -> +websocket_handle({binary, Data}, State = #state{parse_state = ParseState, + proto_state = ProtoState}) -> ?LOG(debug, "RECV ~p", [Data]), BinSize = iolist_size(Data), emqx_pd:update_counter(recv_oct, BinSize), emqx_metrics:trans(inc, 'bytes/received', BinSize), - try emqx_frame:parse(iolist_to_binary(Data), ParserState) of - {more, ParserState1} -> - {ok, State#state{parser_state = ParserState1}}; + try emqx_frame:parse(iolist_to_binary(Data), ParseState) of + {more, ParseState1} -> + {ok, State#state{parse_state = ParseState1}}; {ok, Packet, Rest} -> emqx_metrics:received(Packet), emqx_pd:update_counter(recv_cnt, 1), @@ -248,10 +247,10 @@ websocket_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> {ok, KeepAlive1} -> {ok, State#state{keepalive = KeepAlive1}}; {error, timeout} -> - ?LOG(debug, "Keepalive Timeout!", []), + ?LOG(debug, "Keepalive Timeout!"), shutdown(keepalive_timeout, State); {error, Error} -> - ?LOG(warning, "Keepalive error - ~p", [Error]), + ?LOG(error, "Keepalive error - ~p", [Error]), shutdown(keepalive_error, State) end; @@ -277,15 +276,14 @@ terminate(SockError, _Req, #state{keepalive = Keepalive, proto_state = ProtoState, shutdown = Shutdown}) -> - ?LOG(debug, "Terminated for ~p, sockerror: ~p", - [Shutdown, SockError]), + ?LOG(debug, "Terminated for ~p, sockerror: ~p", [Shutdown, SockError]), emqx_keepalive:cancel(Keepalive), case {ProtoState, Shutdown} of {undefined, _} -> ok; {_, {shutdown, Reason}} -> - emqx_protocol:shutdown(Reason, ProtoState); + emqx_protocol:terminate(Reason, ProtoState); {_, Error} -> - emqx_protocol:shutdown(Error, ProtoState) + emqx_protocol:terminate(Error, ProtoState) end. %%------------------------------------------------------------------------------ @@ -293,7 +291,7 @@ terminate(SockError, _Req, #state{keepalive = Keepalive, %%------------------------------------------------------------------------------ reset_parser(State = #state{proto_state = ProtoState}) -> - State#state{parser_state = emqx_protocol:parser(ProtoState)}. + State#state{parse_state = emqx_protocol:parser(ProtoState)}. ensure_stats_timer(State = #state{enable_stats = true, stats_timer = undefined, diff --git a/test/emqx_alarm_handler_SUITE.erl b/test/emqx_alarm_handler_SUITE.erl new file mode 100644 index 000000000..e58385ebb --- /dev/null +++ b/test/emqx_alarm_handler_SUITE.erl @@ -0,0 +1,145 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_alarm_handler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_mqtt.hrl"). +-include("emqx.hrl"). + +all() -> [t_alarm_handler, t_logger_handler]. + +init_per_suite(Config) -> + [start_apps(App, {SchemaFile, ConfigFile}) || + {App, SchemaFile, ConfigFile} + <- [{emqx, local_path("priv/emqx.schema"), + local_path("etc/emqx.conf")}]], + Config. + +end_per_suite(_Config) -> + application:stop(emqx). + +local_path(RelativePath) -> + filename:join([get_base_dir(), RelativePath]). + +get_base_dir() -> + {file, Here} = code:is_loaded(?MODULE), + filename:dirname(filename:dirname(Here)). + +start_apps(App, {SchemaFile, ConfigFile}) -> + read_schema_configs(App, {SchemaFile, ConfigFile}), + set_special_configs(App), + application:ensure_all_started(App). + +read_schema_configs(App, {SchemaFile, ConfigFile}) -> + ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = conf_parse:file(ConfigFile), + NewConfig = cuttlefish_generator:map(Schema, Conf), + Vals = proplists:get_value(App, NewConfig, []), + [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. + +set_special_configs(_App) -> + ok. + +with_connection(DoFun) -> + {ok, Sock} = emqx_client_sock:connect({127, 0, 0, 1}, 1883, + [binary, {packet, raw}, {active, false}], + 3000), + try + DoFun(Sock) + after + emqx_client_sock:close(Sock) + end. + +t_alarm_handler(_) -> + with_connection( + fun(Sock) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5), + + Topic1 = emqx_topic:systop(<<"alarms/alarm_for_test/alert">>), + Topic2 = emqx_topic:systop(<<"alarms/alarm_for_test/clear">>), + SubOpts = #{rh => 1, qos => ?QOS_2, rap => 0, nl => 0, rc => 0}, + emqx_client_sock:send(Sock, + raw_send_serialize( + ?SUBSCRIBE_PACKET( + 1, + [{Topic1, SubOpts}, + {Topic2, SubOpts}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2, 2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5), + + alarm_handler:set_alarm({alarm_for_test, #alarm{id = alarm_for_test, + severity = error, + title="alarm title", + summary="alarm summary"}}), + + {ok, Data3} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBLISH_PACKET(?QOS_0, Topic1, _, _), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5), + + ?assertEqual(true, lists:keymember(alarm_for_test, 1, emqx_alarm_handler:get_alarms())), + + alarm_handler:clear_alarm(alarm_for_test), + + {ok, Data4} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBLISH_PACKET(?QOS_0, Topic2, _, _), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5), + + ?assertEqual(false, lists:keymember(alarm_for_test, 1, emqx_alarm_handler:get_alarms())) + + end). + +t_logger_handler(_) -> + %% Meck supervisor report + logger:log(error, #{label => {supervisor, start_error}, + report => [{supervisor, {local, tmp_sup}}, + {errorContext, shutdown}, + {reason, reached_max_restart_intensity}, + {offender, [{pid, meck}, + {id, meck}, + {mfargs, {meck, start_link, []}}, + {restart_type, permanent}, + {shutdown, 5000}, + {child_type, worker}]}]}, + #{logger_formatter => #{title => "SUPERVISOR REPORT"}, + report_cb => fun logger:format_otp_report/1}), + ?assertEqual(true, lists:keymember(supervisor_report, 1, emqx_alarm_handler:get_alarms())). + +raw_send_serialize(Packet) -> + emqx_frame:serialize(Packet). + +raw_send_serialize(Packet, Opts) -> + emqx_frame:serialize(Packet, Opts). + +raw_recv_parse(P, ProtoVersion) -> + emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE, + version => ProtoVersion}}). + diff --git a/test/emqx_bridge_SUITE.erl b/test/emqx_bridge_SUITE.erl index 9681c27e6..238871d05 100644 --- a/test/emqx_bridge_SUITE.erl +++ b/test/emqx_bridge_SUITE.erl @@ -14,45 +14,183 @@ -module(emqx_bridge_SUITE). --compile(export_all). --compile(nowarn_export_all). +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([t_rpc/1, + t_mqtt/1, + t_mngr/1 + ]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include("emqx_mqtt.hrl"). +-include("emqx.hrl"). -all() -> - [bridge_test]. +-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). + +all() -> [t_rpc, + t_mqtt, + t_mngr]. init_per_suite(Config) -> - emqx_ct_broker_helpers:run_setup_steps(), - Config. + case node() of + nonode@nohost -> + net_kernel:start(['emqx@127.0.0.1', longnames]); + _ -> + ok + end, + emqx_ct_broker_helpers:run_setup_steps([{log_level, error} | Config]). end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). -bridge_test(_) -> - #{msg := <<"start bridge successfully">>} - = emqx_bridge:start_bridge(aws), - test_forwards(), - test_subscriptions(0), - test_subscriptions(1), - test_subscriptions(2), - #{msg := <<"stop bridge successfully">>} - = emqx_bridge:stop_bridge(aws), +t_mngr(Config) when is_list(Config) -> + Subs = [{<<"a">>, 1}, {<<"b">>, 2}], + Cfg = #{address => node(), + forwards => [<<"mngr">>], + connect_module => emqx_bridge_rpc, + mountpoint => <<"forwarded">>, + subscriptions => Subs, + start_type => auto + }, + Name = ?FUNCTION_NAME, + {ok, Pid} = emqx_bridge:start_link(Name, Cfg), + try + ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr")), + ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr2")), + ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_bridge:get_forwards(Pid)), + ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr2")), + ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr3")), + ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Pid)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_bridge:ensure_subscription_present(Pid, <<"t">>, 0)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_bridge:ensure_subscription_absent(Pid, <<"t">>)), + ?assertEqual(Subs, emqx_bridge:get_subscriptions(Pid)) + after + ok = emqx_bridge:stop(Pid) + end. + +%% A loopback RPC to local node +t_rpc(Config) when is_list(Config) -> + Cfg = #{address => node(), + forwards => [<<"t_rpc/#">>], + connect_module => emqx_bridge_rpc, + mountpoint => <<"forwarded">>, + start_type => auto + }, + {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg), + ClientId = <<"ClientId">>, + try + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + %% message from a different client, to avoid getting terminated by no-local + Msg1 = emqx_message:make(<<"ClientId-2">>, ?QOS_2, <<"t_rpc/one">>, <<"hello">>), + ok = emqx_session:subscribe(SPid, [{<<"forwarded/t_rpc/one">>, #{qos => ?QOS_1}}]), + PacketId = 1, + emqx_session:publish(SPid, PacketId, Msg1), + ?wait(case emqx_mock_client:get_last_message(ConnPid) of + [{publish, PacketId, #message{topic = <<"forwarded/t_rpc/one">>}}] -> + true; + Other -> + Other + end, 4000), + emqx_mock_client:close_session(ConnPid) + after + ok = emqx_bridge:stop(Pid) + end. + +%% Full data loopback flow explained: +%% test-pid ---> mock-cleint ----> local-broker ---(local-subscription)---> +%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> +%% bridge(import) --(mecked message sending)--> test-pid +t_mqtt(Config) when is_list(Config) -> + SendToTopic = <<"t_mqtt/one">>, + SendToTopic2 = <<"t_mqtt/two">>, + Mountpoint = <<"forwarded/${node}/">>, + ForwardedTopic = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic]), + ForwardedTopic2 = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic2]), + Cfg = #{address => "127.0.0.1:1883", + forwards => [SendToTopic], + connect_module => emqx_bridge_mqtt, + mountpoint => Mountpoint, + username => "user", + clean_start => true, + client_id => "bridge_aws", + keepalive => 60000, + max_inflight => 32, + password => "passwd", + proto_ver => mqttv4, + queue => #{replayq_dir => "data/t_mqtt/", + replayq_seg_bytes => 10000, + batch_bytes_limit => 1000, + batch_count_limit => 10 + }, + reconnect_delay_ms => 1000, + ssl => false, + %% Consume back to forwarded message for verification + %% NOTE: this is a indefenite loopback without mocking emqx_bridge:import_batch/2 + subscriptions => [{ForwardedTopic, _QoS = 1}], + start_type => auto + }, + Tester = self(), + Ref = make_ref(), + meck:new(emqx_bridge, [passthrough, no_history]), + meck:expect(emqx_bridge, import_batch, 2, + fun(Batch, AckFun) -> + Tester ! {Ref, Batch}, + AckFun() + end), + {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg), + ClientId = <<"client-1">>, + try + ?assertEqual([{ForwardedTopic, 1}], emqx_bridge:get_subscriptions(Pid)), + ok = emqx_bridge:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), + ok = emqx_bridge:ensure_forward_present(Pid, SendToTopic2), + ?assertEqual([{ForwardedTopic, 1}, + {ForwardedTopic2, 1}], emqx_bridge:get_subscriptions(Pid)), + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + %% message from a different client, to avoid getting terminated by no-local + Max = 100, + Msgs = lists:seq(1, Max), + lists:foreach(fun(I) -> + Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic, integer_to_binary(I)), + emqx_session:publish(SPid, I, Msg) + end, Msgs), + ok = receive_and_match_messages(Ref, Msgs), + Msgs2 = lists:seq(Max + 1, Max * 2), + lists:foreach(fun(I) -> + Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic2, integer_to_binary(I)), + emqx_session:publish(SPid, I, Msg) + end, Msgs2), + ok = receive_and_match_messages(Ref, Msgs2), + emqx_mock_client:close_session(ConnPid) + after + ok = emqx_bridge:stop(Pid), + meck:unload(emqx_bridge) + end. + +receive_and_match_messages(Ref, Msgs) -> + TRef = erlang:send_after(timer:seconds(5), self(), {Ref, timeout}), + try + do_receive_and_match_messages(Ref, Msgs) + after + erlang:cancel_timer(TRef) + end, ok. -test_forwards() -> - emqx_bridge:add_forward(aws, <<"test_forwards">>), - [<<"test_forwards">>, <<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws), - emqx_bridge:del_forward(aws, <<"test_forwards">>), - [<<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws), - ok. - -test_subscriptions(QoS) -> - emqx_bridge:add_subscription(aws, <<"test_subscriptions">>, QoS), - [{<<"test_subscriptions">>, QoS}, - {<<"cmd/topic1">>, 1}, - {<<"cmd/topic2">>, 1}] = emqx_bridge:show_subscriptions(aws), - emqx_bridge:del_subscription(aws, <<"test_subscriptions">>), - [{<<"cmd/topic1">>,1}, {<<"cmd/topic2">>,1}] = emqx_bridge:show_subscriptions(aws), - ok. +do_receive_and_match_messages(_Ref, []) -> ok; +do_receive_and_match_messages(Ref, [I | Rest] = Exp) -> + receive + {Ref, timeout} -> erlang:error(timeout); + {Ref, [#{payload := P} = Msg]} -> + case binary_to_integer(P) of + I -> %% exact match + do_receive_and_match_messages(Ref, Rest); + J when J < I -> %% allow retry + do_receive_and_match_messages(Ref, Exp); + _Other -> + throw({unexpected, Msg, Exp}) + end + end. diff --git a/test/emqx_bridge_mqtt_tests.erl b/test/emqx_bridge_mqtt_tests.erl new file mode 100644 index 000000000..e9eb0b0a0 --- /dev/null +++ b/test/emqx_bridge_mqtt_tests.erl @@ -0,0 +1,54 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_mqtt_tests). +-include_lib("eunit/include/eunit.hrl"). +-include("emqx_mqtt.hrl"). + +send_and_ack_test() -> + %% delegate from gen_rpc to rpc for unit test + meck:new(emqx_client, [passthrough, no_history]), + meck:expect(emqx_client, start_link, 1, + fun(#{msg_handler := Hdlr}) -> + {ok, spawn_link(fun() -> fake_client(Hdlr) end)} + end), + meck:expect(emqx_client, connect, 1, {ok, dummy}), + meck:expect(emqx_client, stop, 1, + fun(Pid) -> Pid ! stop end), + meck:expect(emqx_client, publish, 2, + fun(Client, Msg) -> + Client ! {publish, Msg}, + {ok, Msg} %% as packet id + end), + try + Max = 100, + Batch = lists:seq(1, Max), + {ok, Ref, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}), + %% return last packet id as batch reference + {ok, AckRef} = emqx_bridge_mqtt:send(Conn, Batch), + %% expect batch ack + receive {batch_ack, AckRef} -> ok end, + ok = emqx_bridge_mqtt:stop(Ref, Conn) + after + meck:unload(emqx_client) + end. + +fake_client(#{puback := PubAckCallback} = Hdlr) -> + receive + {publish, PktId} -> + PubAckCallback(#{packet_id => PktId, reason_code => ?RC_SUCCESS}), + fake_client(Hdlr); + stop -> + exit(normal) + end. diff --git a/test/emqx_bridge_rpc_tests.erl b/test/emqx_bridge_rpc_tests.erl new file mode 100644 index 000000000..28e05b895 --- /dev/null +++ b/test/emqx_bridge_rpc_tests.erl @@ -0,0 +1,43 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_rpc_tests). +-include_lib("eunit/include/eunit.hrl"). + +send_and_ack_test() -> + %% delegate from gen_rpc to rpc for unit test + meck:new(gen_rpc, [passthrough, no_history]), + meck:expect(gen_rpc, call, 4, + fun(Node, Module, Fun, Args) -> + rpc:call(Node, Module, Fun, Args) + end), + meck:expect(gen_rpc, cast, 4, + fun(Node, Module, Fun, Args) -> + rpc:cast(Node, Module, Fun, Args) + end), + meck:new(emqx_bridge, [passthrough, no_history]), + meck:expect(emqx_bridge, import_batch, 2, + fun(batch, AckFun) -> AckFun() end), + try + {ok, Pid, Node} = emqx_bridge_rpc:start(#{address => node()}), + {ok, Ref} = emqx_bridge_rpc:send(Node, batch), + receive + {batch_ack, Ref} -> + ok + end, + ok = emqx_bridge_rpc:stop(Pid, Node) + after + meck:unload(gen_rpc), + meck:unload(emqx_bridge) + end. diff --git a/test/emqx_bridge_tests.erl b/test/emqx_bridge_tests.erl new file mode 100644 index 000000000..22b2c4d49 --- /dev/null +++ b/test/emqx_bridge_tests.erl @@ -0,0 +1,157 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_tests). +-behaviour(emqx_bridge_connect). + +-include_lib("eunit/include/eunit.hrl"). +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-define(BRIDGE_NAME, test). +-define(BRIDGE_REG_NAME, emqx_bridge_test). +-define(WAIT(PATTERN, TIMEOUT), + receive + PATTERN -> + ok + after + TIMEOUT -> + error(timeout) + end). + +%% stub callbacks +-export([start/1, send/2, stop/2]). + +start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) -> + case is_pid(Pid) of + true -> Pid ! {connection_start_attempt, Ref}; + false -> ok + end, + Result. + +send(SendFun, Batch) when is_function(SendFun, 1) -> + SendFun(Batch). + +stop(_Ref, _Pid) -> ok. + +%% bridge worker should retry connecting remote node indefinitely +reconnect_test() -> + Ref = make_ref(), + Config = make_config(Ref, self(), {error, test}), + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), + %% assert name registered + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), + ?WAIT({connection_start_attempt, Ref}, 1000), + %% expect same message again + ?WAIT({connection_start_attempt, Ref}, 1000), + ok = emqx_bridge:stop(?BRIDGE_REG_NAME), + ok. + +%% connect first, disconnect, then connect again +disturbance_test() -> + Ref = make_ref(), + Config = make_config(Ref, self(), {ok, Ref, connection}), + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), + ?WAIT({connection_start_attempt, Ref}, 1000), + Pid ! {disconnected, Ref, test}, + ?WAIT({connection_start_attempt, Ref}, 1000), + ok = emqx_bridge:stop(?BRIDGE_REG_NAME). + +%% buffer should continue taking in messages when disconnected +buffer_when_disconnected_test_() -> + {timeout, 10000, fun test_buffer_when_disconnected/0}. + +test_buffer_when_disconnected() -> + Ref = make_ref(), + Nums = lists:seq(1, 100), + Sender = spawn_link(fun() -> receive {bridge, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end), + SenderMref = monitor(process, Sender), + Receiver = spawn_link(fun() -> receive {bridge, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end), + ReceiverMref = monitor(process, Receiver), + SendFun = fun(Batch) -> + BatchRef = make_ref(), + Receiver ! {batch, BatchRef, Batch}, + {ok, BatchRef} + end, + Config0 = make_config(Ref, false, {ok, Ref, SendFun}), + Config = Config0#{reconnect_delay_ms => 100}, + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), + Sender ! {bridge, Pid}, + Receiver ! {bridge, Pid}, + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), + Pid ! {disconnected, Ref, test}, + ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000), + ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), + ok = emqx_bridge:stop(?BRIDGE_REG_NAME). + +manual_start_stop_test() -> + Ref = make_ref(), + Config0 = make_config(Ref, self(), {ok, Ref, connection}), + Config = Config0#{start_type := manual}, + {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config), + %% call ensure_started again should yeld the same result + {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config), + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), + ?assertEqual({error, standing_by}, + emqx_bridge:ensure_forward_present(Pid, "dummy")), + emqx_bridge:ensure_stopped(unknown), + emqx_bridge:ensure_stopped(Pid), + emqx_bridge:ensure_stopped(?BRIDGE_REG_NAME). + +%% Feed messages to bridge +sender_loop(_Pid, [], _) -> exit(normal); +sender_loop(Pid, [Num | Rest], Interval) -> + random_sleep(Interval), + Pid ! {dispatch, dummy, make_msg(Num)}, + sender_loop(Pid, Rest, Interval). + +%% Feed acknowledgments to bridge +receiver_loop(_Pid, [], _) -> ok; +receiver_loop(Pid, Nums, Interval) -> + receive + {batch, BatchRef, Batch} -> + Rest = match_nums(Batch, Nums), + random_sleep(Interval), + emqx_bridge:handle_ack(Pid, BatchRef), + receiver_loop(Pid, Rest, Interval) + end. + +random_sleep(MaxInterval) -> + case rand:uniform(MaxInterval) - 1 of + 0 -> ok; + T -> timer:sleep(T) + end. + +match_nums([], Rest) -> Rest; +match_nums([#message{payload = P} | Rest], Nums) -> + I = binary_to_integer(P), + case Nums of + [I | NumsLeft] -> match_nums(Rest, NumsLeft); + [J | _] when J > I -> match_nums(Rest, Nums); %% allow retry + _ -> error([{received, I}, {expecting, Nums}]) + end. + +make_config(Ref, TestPid, Result) -> + #{test_pid => TestPid, + test_ref => Ref, + connect_module => ?MODULE, + reconnect_delay_ms => 50, + connect_result => Result, + start_type => auto + }. + +make_msg(I) -> + Payload = integer_to_binary(I), + emqx_message:make(<<"test/topic">>, Payload). diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 81796dbdc..e479bbaba 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -29,8 +29,7 @@ all() -> [{group, pubsub}, {group, session}, {group, metrics}, - {group, stats}, - {group, alarms}]. + {group, stats}]. groups() -> [ @@ -41,8 +40,7 @@ groups() -> 'pubsub#', 'pubsub+']}, {session, [sequence], [start_session]}, {metrics, [sequence], [inc_dec_metric]}, - {stats, [sequence], [set_get_stat]}, - {alarms, [sequence], [set_alarms]} + {stats, [sequence], [set_get_stat]} ]. init_per_suite(Config) -> @@ -171,12 +169,3 @@ inc_dec_metric(_) -> set_get_stat(_) -> emqx_stats:setstat('retained/max', 99), 99 = emqx_stats:getstat('retained/max'). - -set_alarms(_) -> - AlarmTest = #alarm{id = <<"1">>, severity = error, title="alarm title", summary="alarm summary"}, - emqx_alarm_mgr:set_alarm(AlarmTest), - Alarms = emqx_alarm_mgr:get_alarms(), - ct:log("Alarms Length: ~p ~n", [length(Alarms)]), - ?assertEqual(1, length(Alarms)), - emqx_alarm_mgr:clear_alarm(<<"1">>), - [] = emqx_alarm_mgr:get_alarms(). diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index aca650215..9c9c3ab55 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -23,59 +23,6 @@ -include("emqx_mqtt.hrl"). --define(STATS, [{mailbox_len, _}, - {heap_size, _}, - {reductions, _}, - {recv_pkt, _}, - {recv_msg, _}, - {send_pkt, _}, - {send_msg, _}, - {recv_oct, _}, - {recv_cnt, _}, - {send_oct, _}, - {send_cnt, _}, - {send_pend, _}]). - --define(ATTRS, [{clean_start, _}, - {client_id, _}, - {connected_at, _}, - {is_bridge, _}, - {is_super, _}, - {keepalive, _}, - {mountpoint, _}, - {peercert, _}, - {peername, _}, - {proto_name, _}, - {proto_ver, _}, - {sockname, _}, - {username, _}, - {zone, _}]). - --define(INFO, [{ack_props, _}, - {active_n, _}, - {clean_start, _}, - {client_id, _}, - {conn_props, _}, - {conn_state, _}, - {connected_at, _}, - {enable_acl, _}, - {is_bridge, _}, - {is_super, _}, - {keepalive, _}, - {mountpoint, _}, - {peercert, _}, - {peername, _}, - {proto_name, _}, - {proto_ver, _}, - {pub_limit, _}, - {rate_limit, _}, - {session, _}, - {sockname, _}, - {socktype, _}, - {topic_aliases, _}, - {username, _}, - {zone, _}]). - all() -> [t_connect_api]. @@ -93,9 +40,33 @@ t_connect_api(_Config) -> {password, <<"pass1">>}]), {ok, _} = emqx_client:connect(T1), CPid = emqx_cm:lookup_conn_pid(<<"client1">>), - ?STATS = emqx_connection:stats(CPid), - ?ATTRS = emqx_connection:attrs(CPid), - ?INFO = emqx_connection:info(CPid), + ConnStats = emqx_connection:stats(CPid), + ok = t_stats(ConnStats), + ConnAttrs = emqx_connection:attrs(CPid), + ok = t_attrs(ConnAttrs), + ConnInfo = emqx_connection:info(CPid), + ok = t_info(ConnInfo), SessionPid = emqx_connection:session(CPid), true = is_pid(SessionPid), emqx_client:disconnect(T1). + +t_info(ConnInfo) -> + ?assertEqual(tcp, proplists:get_value(socktype, ConnInfo)), + ?assertEqual(running, proplists:get_value(conn_state, ConnInfo)), + ?assertEqual(<<"client1">>, proplists:get_value(client_id, ConnInfo)), + ?assertEqual(<<"testuser1">>, proplists:get_value(username, ConnInfo)), + ?assertEqual(<<"MQTT">>, proplists:get_value(proto_name, ConnInfo)). + +t_attrs(AttrsData) -> + ?assertEqual(<<"client1">>, proplists:get_value(client_id, AttrsData)), + ?assertEqual(emqx_connection, proplists:get_value(conn_mod, AttrsData)), + ?assertEqual(<<"testuser1">>, proplists:get_value(username, AttrsData)). + +t_stats(StatsData) -> + ?assertEqual(true, proplists:get_value(recv_oct, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(mailbox_len, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(heap_size, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(reductions, StatsData) >=0), + ?assertEqual(true, proplists:get_value(recv_pkt, StatsData) =:=1), + ?assertEqual(true, proplists:get_value(recv_msg, StatsData) >=0), + ?assertEqual(true, proplists:get_value(send_pkt, StatsData) =:=1). diff --git a/test/emqx_ct_broker_helpers.erl b/test/emqx_ct_broker_helpers.erl index 1ab79e8a9..88240be85 100644 --- a/test/emqx_ct_broker_helpers.erl +++ b/test/emqx_ct_broker_helpers.erl @@ -54,10 +54,17 @@ "ECDH-RSA-AES128-SHA","AES128-SHA"]}]). run_setup_steps() -> + _ = run_setup_steps([]), + %% return ok to be backward compatible + ok. + +run_setup_steps(Config) -> NewConfig = generate_config(), lists:foreach(fun set_app_env/1, NewConfig), set_bridge_env(), - application:ensure_all_started(?APP). + {ok, _} = application:ensure_all_started(?APP), + set_log_level(Config), + Config. run_teardown_steps() -> ?APP:shutdown(). @@ -67,6 +74,12 @@ generate_config() -> Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), cuttlefish_generator:map(Schema, Conf). +set_log_level(Config) -> + case proplists:get_value(log_level, Config) of + undefined -> ok; + Level -> emqx_logger:set_log_level(Level) + end. + get_base_dir(Module) -> {file, Here} = code:is_loaded(Module), filename:dirname(filename:dirname(Here)). @@ -156,24 +169,30 @@ flush(Msgs) -> end. bridge_conf() -> - [{aws, - [{username,"user"}, - {address,"127.0.0.1:1883"}, - {clean_start,true}, - {client_id,"bridge_aws"}, - {forwards,["topic1/#","topic2/#"]}, - {keepalive,60000}, - {max_inflight,32}, - {mountpoint,"bridge/aws/${node}/"}, - {password,"passwd"}, - {proto_ver,mqttv4}, - {queue, - #{batch_size => 1000,mem_cache => true, - replayq_dir => "data/emqx_aws_bridge/", - replayq_seg_bytes => 10485760}}, - {reconnect_interval,30000}, - {retry_interval,20000}, - {ssl,false}, - {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]}, - {start_type,manual}, - {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}]. \ No newline at end of file + [ {local_rpc, + [{connect_module, emqx_bridge_rpc}, + {address, node()}, + {forwards, ["bridge-1/#", "bridge-2/#"]} + ]} + ]. + % [{aws, + % [{connect_module, emqx_bridge_mqtt}, + % {username,"user"}, + % {address,"127.0.0.1:1883"}, + % {clean_start,true}, + % {client_id,"bridge_aws"}, + % {forwards,["topic1/#","topic2/#"]}, + % {keepalive,60000}, + % {max_inflight,32}, + % {mountpoint,"bridge/aws/${node}/"}, + % {password,"passwd"}, + % {proto_ver,mqttv4}, + % {queue, + % #{batch_coun t_limit => 1000, + % replayq_dir => "data/emqx_aws_bridge/", + % replayq_seg_bytes => 10485760}}, + % {reconnect_delay_ms,30000}, + % {ssl,false}, + % {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]}, + % {start_type,manual}, + % {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}]. diff --git a/test/emqx_ct_helpers.erl b/test/emqx_ct_helpers.erl index eae22d6ab..2f35f42cc 100644 --- a/test/emqx_ct_helpers.erl +++ b/test/emqx_ct_helpers.erl @@ -14,9 +14,55 @@ -module(emqx_ct_helpers). --export([ensure_mnesia_stopped/0]). +-export([ensure_mnesia_stopped/0, wait_for/4]). ensure_mnesia_stopped() -> ekka_mnesia:ensure_stopped(), ekka_mnesia:delete_schema(). +%% Help function to wait for Fun to yield 'true'. +wait_for(Fn, Ln, F, Timeout) -> + {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), + wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). + +wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> + receive + {'DOWN', Mref, process, Pid, normal} -> + ok; + {'DOWN', Mref, process, Pid, {unexpected, Result}} -> + erlang:error({unexpected, Fn, Ln, Result}); + {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> + erlang:raise(C, {Fn, Ln, E}, S) + after + Timeout -> + case Kill of + true -> + erlang:demonitor(Mref, [flush]), + erlang:exit(Pid, kill), + erlang:error({Fn, Ln, timeout}); + false -> + Pid ! stop, + wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) + end + end. + +wait_loop(_F, ok) -> exit(normal); +wait_loop(F, LastRes) -> + receive + stop -> erlang:exit(LastRes) + after + 100 -> + Res = catch_call(F), + wait_loop(F, Res) + end. + +catch_call(F) -> + try + case F() of + true -> ok; + Other -> {unexpected, Other} + end + catch + C : E : S -> + {crashed, {C, E, S}} + end. diff --git a/test/emqx_misc_tests.erl b/test/emqx_misc_tests.erl index d0da45fba..038180b5b 100644 --- a/test/emqx_misc_tests.erl +++ b/test/emqx_misc_tests.erl @@ -47,12 +47,14 @@ timer_cancel_flush_test() -> end. shutdown_disabled_test() -> + ok = drain(), self() ! foo, ?assertEqual(continue, conn_proc_mng_policy(0)), receive foo -> ok end, ?assertEqual(hibernate, conn_proc_mng_policy(0)). message_queue_too_long_test() -> + ok = drain(), self() ! foo, self() ! bar, ?assertEqual({shutdown, message_queue_too_long}, @@ -63,3 +65,18 @@ message_queue_too_long_test() -> conn_proc_mng_policy(L) -> emqx_misc:conn_proc_mng_policy(#{message_queue_len => L}). + +%% drain self() msg queue for deterministic test behavior +drain() -> + _ = drain([]), % maybe log + ok. + +drain(Acc) -> + receive + Msg -> + drain([Msg | Acc]) + after + 0 -> + lists:reverse(Acc) + end. + diff --git a/test/emqx_mqtt_packet_SUITE.erl b/test/emqx_mqtt_packet_SUITE.erl index 3be2617b0..8ac76bfd9 100644 --- a/test/emqx_mqtt_packet_SUITE.erl +++ b/test/emqx_mqtt_packet_SUITE.erl @@ -1,18 +1,16 @@ -%%%=================================================================== -%%% Copyright (c) 2013-2013-2019 EMQ Inc. All rights reserved. -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%%=================================================================== +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. -module(emqx_mqtt_packet_SUITE). @@ -90,7 +88,7 @@ case1_protocol_name(_) -> {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data), Disconnect = gen_tcp:recv(Sock, 0), ?assertEqual({error, closed}, Disconnect). - + case2_protocol_ver(_) -> {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), Packet = serialize(?CASE2_PROTOCAL_VER), diff --git a/test/emqx_os_mon_SUITE.erl b/test/emqx_os_mon_SUITE.erl new file mode 100644 index 000000000..67a3959a5 --- /dev/null +++ b/test/emqx_os_mon_SUITE.erl @@ -0,0 +1,56 @@ +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_os_mon_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +all() -> [t_api]. + +init_per_suite(Config) -> + application:ensure_all_started(os_mon), + Config. + +end_per_suite(_Config) -> + application:stop(os_mon). + +t_api(_) -> + gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), + {ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1}, + {cpu_high_watermark, 0.05}, + {cpu_low_watermark, 0.80}, + {mem_check_interval, 60}, + {sysmem_high_watermark, 0.70}, + {procmem_high_watermark, 0.05}]), + ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()), + ?assertEqual(0.05, emqx_os_mon:get_cpu_high_watermark()), + ?assertEqual(0.80, emqx_os_mon:get_cpu_low_watermark()), + ?assertEqual(60, emqx_os_mon:get_mem_check_interval()), + ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()), + ?assertEqual(0.05, emqx_os_mon:get_procmem_high_watermark()), + % timer:sleep(2000), + % ?assertEqual(true, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())), + + emqx_os_mon:set_cpu_high_watermark(0.8), + emqx_os_mon:set_cpu_low_watermark(0.75), + ?assertEqual(0.8, emqx_os_mon:get_cpu_high_watermark()), + ?assertEqual(0.75, emqx_os_mon:get_cpu_low_watermark()), + % timer:sleep(3000), + % ?assertEqual(false, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())), + ok. diff --git a/test/emqx_pool_SUITE.erl b/test/emqx_pool_SUITE.erl index ea648709f..752e41a12 100644 --- a/test/emqx_pool_SUITE.erl +++ b/test/emqx_pool_SUITE.erl @@ -62,7 +62,7 @@ async_submit_mfa(_Config) -> emqx_pool:async_submit(fun ?MODULE:test_mfa/0, []). async_submit_crash(_) -> - emqx_pool:async_submit(fun() -> A = 1, A = 0 end). + emqx_pool:async_submit(fun() -> error(unexpected_error) end). t_unexpected(_) -> Pid = emqx_pool:worker(), @@ -73,3 +73,4 @@ t_unexpected(_) -> test_mfa() -> lists:foldl(fun(X, Sum) -> X + Sum end, 0, [1,2,3,4,5]). + diff --git a/test/emqx_protocol_SUITE.erl b/test/emqx_protocol_SUITE.erl index e48825d76..a2d314acc 100644 --- a/test/emqx_protocol_SUITE.erl +++ b/test/emqx_protocol_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2013-2019 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,7 +11,6 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -module(emqx_protocol_SUITE). @@ -33,64 +31,66 @@ username = <<"emqx">>, password = <<"public">>})). --record(pstate, { - zone, - sendfun, - peername, - peercert, - proto_ver, - proto_name, - client_id, - is_assigned, - conn_pid, - conn_props, - ack_props, - username, - session, - clean_start, - topic_aliases, - packet_size, - keepalive, - mountpoint, - is_super, - is_bridge, - enable_ban, - enable_acl, - acl_deny_action, - recv_stats, - send_stats, - connected, - connected_at, - ignore_loop, - topic_alias_maximum - }). +% -record(pstate, { +% zone, +% sendfun, +% peername, +% peercert, +% proto_ver, +% proto_name, +% client_id, +% is_assigned, +% conn_pid, +% conn_props, +% ack_props, +% username, +% session, +% clean_start, +% topic_aliases, +% packet_size, +% keepalive, +% mountpoint, +% is_super, +% is_bridge, +% enable_ban, +% enable_acl, +% acl_deny_action, +% recv_stats, +% send_stats, +% connected, +% connected_at, +% ignore_loop, +% topic_alias_maximum, +% conn_mod +% }). --define(TEST_PSTATE(ProtoVer, SendStats), - #pstate{zone = test, - sendfun = fun(_Packet, _Options) -> ok end, - peername = test_peername, - peercert = test_peercert, - proto_ver = ProtoVer, - proto_name = <<"MQTT">>, - client_id = <<"test_pstate">>, - is_assigned = false, - conn_pid = self(), - username = <<"emqx">>, - is_super = false, - clean_start = false, - topic_aliases = #{}, - packet_size = 1000, - mountpoint = <<>>, - is_bridge = false, - enable_ban = false, - enable_acl = true, - acl_deny_action = disconnect, - recv_stats = #{msg => 0, pkt => 0}, - send_stats = SendStats, - connected = false, - ignore_loop = false, - topic_alias_maximum = #{to_client => 0, from_client => 0}}). +% -define(TEST_PSTATE(ProtoVer, SendStats), +% #pstate{zone = test, +% sendfun = fun(_Packet, _Options) -> ok end, +% peername = test_peername, +% peercert = test_peercert, +% proto_ver = ProtoVer, +% proto_name = <<"MQTT">>, +% client_id = <<"test_pstate">>, +% is_assigned = false, +% conn_pid = self(), +% username = <<"emqx">>, +% is_super = false, +% clean_start = false, +% topic_aliases = #{}, +% packet_size = 1000, +% mountpoint = <<>>, +% is_bridge = false, +% enable_ban = false, +% enable_acl = true, +% acl_deny_action = disconnect, +% recv_stats = #{msg => 0, pkt => 0}, +% send_stats = SendStats, +% connected = false, +% ignore_loop = false, +% topic_alias_maximum = #{to_client => 0, from_client => 0}, +% conn_mod = emqx_connection}). all() -> [ @@ -112,8 +112,7 @@ groups() -> [connect_v5, subscribe_v5]}, {acl, [sequence], - [acl_deny_action_ct, - acl_deny_action_eunit]}]. + [acl_deny_action_ct]}]. init_per_suite(Config) -> [start_apps(App, SchemaFile, ConfigFile) || @@ -571,13 +570,13 @@ acl_deny_action_ct(_) -> emqx_zone:set_env(external, acl_deny_action, ignore), ok. -acl_deny_action_eunit(_) -> - PState = ?TEST_PSTATE(?MQTT_PROTO_V5, #{msg => 0, pkt => 0}), - CodeName = emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ?MQTT_PROTO_V5), - {error, CodeName, NEWPSTATE1} = emqx_protocol:process_packet(?PUBLISH_PACKET(?QOS_1, <<"acl_deny_action">>, 1, <<"payload">>), PState), - ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE1#pstate.send_stats), - {error, CodeName, NEWPSTATE2} = emqx_protocol:process_packet(?PUBLISH_PACKET(?QOS_2, <<"acl_deny_action">>, 2, <<"payload">>), PState), - ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE2#pstate.send_stats). +% acl_deny_action_eunit(_) -> +% PState = ?TEST_PSTATE(?MQTT_PROTO_V5, #{msg => 0, pkt => 0}), +% CodeName = emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ?MQTT_PROTO_V5), +% {error, CodeName, NEWPSTATE1} = emqx_protocol:process(?PUBLISH_PACKET(?QOS_1, <<"acl_deny_action">>, 1, <<"payload">>), PState), +% ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE1#pstate.send_stats), +% {error, CodeName, NEWPSTATE2} = emqx_protocol:process(?PUBLISH_PACKET(?QOS_2, <<"acl_deny_action">>, 2, <<"payload">>), PState), +% ?assertEqual(#{pkt => 1, msg => 0}, NEWPSTATE2#pstate.send_stats). will_topic_check(_) -> {ok, Client} = emqx_client:start_link([{username, <<"emqx">>}, diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index 54b2e8579..37ce34be9 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -1,4 +1,3 @@ - %% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,7 +44,7 @@ ignore_loop(_Config) -> application:set_env(emqx, mqtt_ignore_loop_deliver, false). t_session_all(_) -> - emqx_zone:set_env(internal, idle_timeout, 100), + emqx_zone:set_env(internal, idle_timeout, 1000), ClientId = <<"ClientId">>, {ok, ConnPid} = emqx_mock_client:start_link(ClientId), {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), @@ -56,7 +55,7 @@ t_session_all(_) -> [{<<"topic">>, _}] = emqx:subscriptions(SPid), emqx_session:publish(SPid, 1, Message1), timer:sleep(200), - {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid), + [{publish, 1, _}] = emqx_mock_client:get_last_message(ConnPid), Attrs = emqx_session:attrs(SPid), Info = emqx_session:info(SPid), Stats = emqx_session:stats(SPid), diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 1ee059812..6cd6c98d0 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -29,7 +29,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(wait(For, Timeout), wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). +-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). all() -> [t_random_basic, t_random, @@ -59,7 +59,7 @@ t_random_basic(_) -> PacketId = 1, emqx_session:publish(SPid, PacketId, Message1), ?wait(case emqx_mock_client:get_last_message(ConnPid) of - {publish, 1, _} -> true; + [{publish, 1, _}] -> true; Other -> Other end, 1000), emqx_session:pubrec(SPid, PacketId, reasoncode), @@ -105,7 +105,7 @@ t_no_connection_nack(_) -> fun(PacketId, ConnPid) -> Payload = MkPayload(PacketId), case emqx_mock_client:get_last_message(ConnPid) of - {publish, _, #message{payload = Payload}} -> + [{publish, _, #message{payload = Payload}}] -> CasePid ! {Ref, PacketId, ConnPid}, true; _Other -> @@ -176,7 +176,7 @@ t_not_so_sticky(_) -> ?wait(subscribed(<<"group1">>, <<"foo/bar">>, SPid1), 1000), emqx_session:publish(SPid1, 1, Message1), ?wait(case emqx_mock_client:get_last_message(ConnPid1) of - {publish, _, #message{payload = <<"hello1">>}} -> true; + [{publish, _, #message{payload = <<"hello1">>}}] -> true; Other -> Other end, 1000), emqx_mock_client:close_session(ConnPid1), @@ -185,7 +185,7 @@ t_not_so_sticky(_) -> ?wait(subscribed(<<"group1">>, <<"foo/#">>, SPid2), 1000), emqx_session:publish(SPid2, 2, Message2), ?wait(case emqx_mock_client:get_last_message(ConnPid2) of - {publish, _, #message{payload = <<"hello2">>}} -> true; + [{publish, _, #message{payload = <<"hello2">>}}] -> true; Other -> Other end, 1000), emqx_mock_client:close_session(ConnPid2), @@ -240,7 +240,7 @@ test_two_messages(Strategy, WithAck) -> last_message(_ExpectedPayload, []) -> <<"not yet?">>; last_message(ExpectedPayload, [Pid | Pids]) -> case emqx_mock_client:get_last_message(Pid) of - {publish, _, #message{payload = ExpectedPayload}} -> {true, Pid}; + [{publish, _, #message{payload = ExpectedPayload}}] -> {true, Pid}; _Other -> last_message(ExpectedPayload, Pids) end. @@ -259,49 +259,3 @@ ensure_config(Strategy, AckEnabled) -> subscribed(Group, Topic, Pid) -> lists:member(Pid, emqx_shared_sub:subscribers(Group, Topic)). -wait_for(Fn, Ln, F, Timeout) -> - {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), - wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). - -wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> - receive - {'DOWN', Mref, process, Pid, normal} -> - ok; - {'DOWN', Mref, process, Pid, {unexpected, Result}} -> - erlang:error({unexpected, Fn, Ln, Result}); - {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> - erlang:raise(C, {Fn, Ln, E}, S) - after - Timeout -> - case Kill of - true -> - erlang:demonitor(Mref, [flush]), - erlang:exit(Pid, kill), - erlang:error({Fn, Ln, timeout}); - false -> - Pid ! stop, - wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) - end - end. - -wait_loop(_F, ok) -> exit(normal); -wait_loop(F, LastRes) -> - receive - stop -> erlang:exit(LastRes) - after - 100 -> - Res = catch_call(F), - wait_loop(F, Res) - end. - -catch_call(F) -> - try - case F() of - true -> ok; - Other -> {unexpected, Other} - end - catch - C : E : S -> - {crashed, {C, E, S}} - end. - diff --git a/test/emqx_vm_mon_SUITE.erl b/test/emqx_vm_mon_SUITE.erl new file mode 100644 index 000000000..41a717293 --- /dev/null +++ b/test/emqx_vm_mon_SUITE.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2013-2013-2019 EMQ Enterprise, Inc. (http://emqtt.io) +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_vm_mon_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +all() -> [t_api]. + +init_per_suite(Config) -> + application:ensure_all_started(sasl), + Config. + +end_per_suite(_Config) -> + application:stop(sasl). + +t_api(_) -> + gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), + {ok, _} = emqx_vm_mon:start_link([{check_interval, 1}, + {process_high_watermark, 0}, + {process_low_watermark, 0.6}]), + timer:sleep(2000), + ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_process_high_watermark(0.8), + emqx_vm_mon:set_process_low_watermark(0.75), + ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()), + ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()), + timer:sleep(3000), + ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_check_interval(20), + ?assertEqual(20, emqx_vm_mon:get_check_interval()), + ok. diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index c3698861b..c45344bae 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -35,56 +35,6 @@ -define(PUBQOS, 1). --define(INFO, [{socktype, _}, - {conn_state, _}, - {peername, _}, - {sockname, _}, - {zone, _}, - {client_id, <<"mqtt_client">>}, - {username, <<"admin">>}, - {peername, _}, - {peercert, _}, - {proto_ver, _}, - {proto_name, _}, - {clean_start, _}, - {keepalive, _}, - {mountpoint, _}, - {is_super, _}, - {is_bridge, _}, - {connected_at, _}, - {conn_props, _}, - {ack_props, _}, - {session, _}, - {topic_aliases, _}, - {enable_acl, _}]). - --define(ATTRS, [{clean_start,true}, - {client_id, <<"mqtt_client">>}, - {connected_at, _}, - {is_bridge, _}, - {is_super, _}, - {keepalive, _}, - {mountpoint, _}, - {peercert, _}, - {peername, _}, - {proto_name, _}, - {proto_ver, _}, - {sockname, _}, - {username, <<"admin">>}, - {zone, _}]). - --define(STATS, [{recv_oct, _}, - {recv_cnt, _}, - {send_oct, _}, - {send_cnt, _}, - {mailbox_len, _}, - {heap_size, _}, - {reductions, _}, - {recv_pkt, _}, - {recv_msg, _}, - {send_pkt, _}, - {send_msg, _}]). - all() -> [t_ws_connect_api]. @@ -103,9 +53,12 @@ t_ws_connect_api(_Config) -> {binary, CONACK} = rfc6455_client:recv(WS), {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(CONACK), Pid = emqx_cm:lookup_conn_pid(<<"mqtt_client">>), - ?INFO = emqx_ws_connection:info(Pid), - ?ATTRS = emqx_ws_connection:attrs(Pid), - ?STATS = emqx_ws_connection:stats(Pid), + ConnInfo = emqx_ws_connection:info(Pid), + ok = t_info(ConnInfo), + ConnAttrs = emqx_ws_connection:attrs(Pid), + ok = t_attrs(ConnAttrs), + ConnStats = emqx_ws_connection:stats(Pid), + ok = t_stats(ConnStats), SessionPid = emqx_ws_connection:session(Pid), true = is_pid(SessionPid), ok = emqx_ws_connection:kick(Pid), @@ -118,3 +71,24 @@ raw_send_serialize(Packet) -> raw_recv_pase(P) -> emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE, version => ?MQTT_PROTO_V4} }). + +t_info(InfoData) -> + ?assertEqual(websocket, proplists:get_value(socktype, InfoData)), + ?assertEqual(running, proplists:get_value(conn_state, InfoData)), + ?assertEqual(<<"mqtt_client">>, proplists:get_value(client_id, InfoData)), + ?assertEqual(<<"admin">>, proplists:get_value(username, InfoData)), + ?assertEqual(<<"MQTT">>, proplists:get_value(proto_name, InfoData)). + +t_attrs(AttrsData) -> + ?assertEqual(<<"mqtt_client">>, proplists:get_value(client_id, AttrsData)), + ?assertEqual(emqx_ws_connection, proplists:get_value(conn_mod, AttrsData)), + ?assertEqual(<<"admin">>, proplists:get_value(username, AttrsData)). + +t_stats(StatsData) -> + ?assertEqual(true, proplists:get_value(recv_oct, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(mailbox_len, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(heap_size, StatsData) >= 0), + ?assertEqual(true, proplists:get_value(reductions, StatsData) >=0), + ?assertEqual(true, proplists:get_value(recv_pkt, StatsData) =:=1), + ?assertEqual(true, proplists:get_value(recv_msg, StatsData) >=0), + ?assertEqual(true, proplists:get_value(send_pkt, StatsData) =:=1). \ No newline at end of file