From fbe67e67842edeeabd765be465ce7456909f31c6 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Mon, 14 Jan 2019 07:38:43 +0100 Subject: [PATCH 01/26] Introduce new bridge impl --- Makefile | 8 +- etc/emqx.conf | 21 +- .../emqx_client.hrl | 18 +- priv/emqx.schema | 85 +++- rebar.config | 3 + src/emqx_bridge.erl | 463 ------------------ src/emqx_bridge_sup.erl | 45 -- src/emqx_client.erl | 6 +- src/emqx_local_bridge.erl | 157 ------ src/emqx_local_bridge_sup_sup.erl | 74 --- src/emqx_mqueue.erl | 2 +- src/emqx_portal_connect.erl | 65 +++ src/emqx_sup.erl | 8 +- src/emqx_topic.erl | 14 +- src/portal/emqx_portal.erl | 356 ++++++++++++++ src/portal/emqx_portal_msg.erl | 61 +++ src/portal/emqx_portal_rpc.erl | 106 ++++ src/portal/emqx_portal_sup.erl | 54 ++ test/emqx_bridge_SUITE.erl | 58 --- test/emqx_ct_broker_helpers.erl | 48 +- test/emqx_pool_SUITE.erl | 2 +- test/emqx_portal_rpc_tests.erl | 43 ++ test/emqx_portal_tests.erl | 146 ++++++ 23 files changed, 956 insertions(+), 887 deletions(-) rename src/emqx_local_bridge_sup.erl => include/emqx_client.hrl (55%) delete mode 100644 src/emqx_bridge.erl delete mode 100644 src/emqx_bridge_sup.erl delete mode 100644 src/emqx_local_bridge.erl delete mode 100644 src/emqx_local_bridge_sup_sup.erl create mode 100644 src/emqx_portal_connect.erl create mode 100644 src/portal/emqx_portal.erl create mode 100644 src/portal/emqx_portal_msg.erl create mode 100644 src/portal/emqx_portal_rpc.erl create mode 100644 src/portal/emqx_portal_sup.erl delete mode 100644 test/emqx_bridge_SUITE.erl create mode 100644 test/emqx_portal_rpc_tests.erl create mode 100644 test/emqx_portal_tests.erl diff --git a/Makefile b/Makefile index 6022dbf8f..f6230d2c7 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ ERLC_OPTS += +debug_info -DAPPLICATION=emqx BUILD_DEPS = cuttlefish dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1 -#TEST_DEPS = emqx_ct_helplers -#dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers +TEST_DEPS = meck +dep_meck = hex-emqx 0.8.13 TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx @@ -35,7 +35,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_keepalive emqx_lib emqx_metrics emqx_mod emqx_mod_sup emqx_mqtt_caps \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ - emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ + emqx_listeners emqx_protocol emqx_pool emqx_shared_sub \ emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message @@ -96,7 +96,7 @@ rebar-deps: @rebar3 get-deps rebar-eunit: $(CUTTLEFISH_SCRIPT) - @rebar3 eunit + @rebar3 eunit -v rebar-compile: @rebar3 compile diff --git a/etc/emqx.conf b/etc/emqx.conf index 166fca25c..5a7a698d7 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1694,14 +1694,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: Number ## bridge.aws.subscription.2.qos = 1 -## If enabled, queue would be written into disk more quickly. -## However, If disabled, some message would be dropped in -## the situation emqx crashed. -## -## Value: on | off -## bridge.aws.queue.mem_cache = on - -## Batch size for buffer queue stored +## Maximum number of messages in one batch for buffer queue to store ## ## Value: Integer ## default: 1000 @@ -1709,9 +1702,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, -## replayq works in a mem-only manner. If the config -## entry was set to `bridge.aws.mqueue_type = memory` -## this config entry would have no effect on mqueue +## replayq works in a mem-only manner. ## ## Value: String ## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ @@ -1861,13 +1852,11 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, -## replayq works in a mem-only manner. If the config -## entry was set to `bridge.aws.mqueue_type = memory` -## this config entry would have no effect on mqueue +## replayq works in a mem-only manner. ## ## Value: String -## Default: {{ platform_data_dir }}/emqx_aws_bridge/ -## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ +## Default: "" +## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_azure.bridge/ ## Replayq segment size ## diff --git a/src/emqx_local_bridge_sup.erl b/include/emqx_client.hrl similarity index 55% rename from src/emqx_local_bridge_sup.erl rename to include/emqx_client.hrl index db349b94d..ce66c98d0 100644 --- a/src/emqx_local_bridge_sup.erl +++ b/include/emqx_client.hrl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,15 +12,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_local_bridge_sup). - --include("emqx.hrl"). - --export([start_link/3]). - --spec(start_link(node(), emqx_topic:topic(), [emqx_local_bridge:option()]) - -> {ok, pid()} | {error, term()}). -start_link(Node, Topic, Options) -> - MFA = {emqx_local_bridge, start_link, [Node, Topic, Options]}, - emqx_pool_sup:start_link({bridge, Node, Topic}, random, MFA). +-ifndef(EMQX_CLIENT_HRL). +-define(EMQX_CLIENT_HRL, true). +-record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, + packet_id, topic, props, payload}). +-endif. diff --git a/priv/emqx.schema b/priv/emqx.schema index 9e71248a7..7082d4f87 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1512,20 +1512,9 @@ end}. %%-------------------------------------------------------------------- %% Bridges %%-------------------------------------------------------------------- -{mapping, "bridge.$name.queue.mem_cache", "emqx.bridges", [ - {datatype, flag} -]}. - -{mapping, "bridge.$name.queue.batch_size", "emqx.bridges", [ - {datatype, integer} -]}. - -{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [ - {datatype, string} -]}. - -{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [ - {datatype, bytesize} +{mapping, "bridge.$name.transport", "emqx.bridges", [ + {default, mqtt_client}, + {datatype, {enum, [emqx_portal, mqtt_client]}} ]}. {mapping, "bridge.$name.address", "emqx.bridges", [ @@ -1611,16 +1600,23 @@ end}. {datatype, {duration, ms}} ]}. -{mapping, "bridge.$name.retry_interval", "emqx.bridges", [ - {default, "20s"}, - {datatype, {duration, ms}} -]}. - {mapping, "bridge.$name.max_inflight", "emqx.bridges", [ {default, 0}, {datatype, integer} ]}. +{mapping, "bridge.$name.queue.batch_size", "emqx.bridges", [ + {datatype, integer} +]}. + +{mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.queue.replayq_seg_bytes", "emqx.bridges", [ + {datatype, bytesize} +]}. + {translation, "emqx.bridges", fun(Conf) -> Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, @@ -1661,17 +1657,58 @@ end}. lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, "subscription", I, "topic"], Topic} <- Configs])], [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, "subscription", I, "qos"], QoS} <- Configs])]) end, - + IsNodeAddr = fun(Addr) -> + case string:tokens(Addr, "@") of + [_NodeName, _Hostname] -> true; + _ -> false + end + end, + ConnMod = fun(Name) -> + [Addr] = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".address", Conf), + Subs = Subscriptions(Name), + case IsNodeAddr(Addr) of + true when Subs =/= [] -> + error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs}); + true -> + emqx_portal_rpc; + false -> + emqx_portal_mqtt + end + end, + %% to be backward compatible + Translate = + fun Tr(queue, Q, Cfg) -> + NewQ = maps:fold(Tr, #{}, Q), + Cfg#{queue => NewQ}; + Tr(address, Addr0, Cfg) -> + Addr = case IsNodeAddr(Addr0) of + true -> list_to_atom(Addr0); + false -> Addr0 + end, + Cfg#{address => Addr}; + Tr(batch_size, Count, Cfg) -> + Cfg#{batch_count_limit => Count}; + Tr(reconnect_interval, Ms, Cfg) -> + Cfg#{reconnect_delay_ms => Ms}; + Tr(max_inflight, Count, Cfg) -> + Cfg#{max_inflight_batches => Count}; + Tr(Key, Value, Cfg) -> + Cfg#{Key => Value} + end, maps:to_list( lists:foldl( fun({["bridge", Name, Opt], Val}, Acc) -> %% e.g #{aws => [{OptKey, OptVal}]} - Init = [{list_to_atom(Opt), Val},{subscriptions, Subscriptions(Name)}, {queue, Queue(Name)}], - maps:update_with(list_to_atom(Name), - fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc); + Init = [{list_to_atom(Opt), Val}, + {connect_module, ConnMod(Name)}, + {subscriptions, Subscriptions(Name)}, + {queue, Queue(Name)} + ], + C = maps:update_with(list_to_atom(Name), + fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc), + maps:fold(Translate, #{}, C); (_, Acc) -> Acc end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf)))) - end}. %%-------------------------------------------------------------------- diff --git a/rebar.config b/rebar.config index 7486e7267..b6c68208b 100644 --- a/rebar.config +++ b/rebar.config @@ -27,3 +27,6 @@ {cover_export_enabled, true}. {plugins, [coveralls]}. + +{profiles, [{test, [{deps, [{meck, "0.8.13"}]}]}]}. + diff --git a/src/emqx_bridge.erl b/src/emqx_bridge.erl deleted file mode 100644 index 1ee5612e6..000000000 --- a/src/emqx_bridge.erl +++ /dev/null @@ -1,463 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_bridge). - --behaviour(gen_server). - --include("emqx.hrl"). --include("emqx_mqtt.hrl"). - --import(proplists, [get_value/2, get_value/3]). - --export([start_link/2, start_bridge/1, stop_bridge/1, status/1]). - --export([show_forwards/1, add_forward/2, del_forward/2]). - --export([show_subscriptions/1, add_subscription/3, del_subscription/2]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --record(state, {client_pid :: pid(), - options :: list(), - reconnect_interval :: pos_integer(), - mountpoint :: binary(), - readq :: list(), - writeq :: list(), - replayq :: map(), - ackref :: replayq:ack_ref(), - queue_option :: map(), - forwards :: list(), - subscriptions :: list()}). - --record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, - packet_id, topic, props, payload}). - -start_link(Name, Options) -> - gen_server:start_link({local, name(Name)}, ?MODULE, [Options], []). - -start_bridge(Name) -> - gen_server:call(name(Name), start_bridge). - -stop_bridge(Name) -> - gen_server:call(name(Name), stop_bridge). - --spec(show_forwards(atom()) -> list()). -show_forwards(Name) -> - gen_server:call(name(Name), show_forwards). - --spec(add_forward(atom(), binary()) -> ok | {error, already_exists | validate_fail}). -add_forward(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {add_forward, Topic}) - catch - _Error:_Reason -> - {error, validate_fail} - end. - --spec(del_forward(atom(), binary()) -> ok | {error, validate_fail}). -del_forward(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {del_forward, Topic}) - catch - _Error:_Reason -> - {error, validate_fail} - end. - --spec(show_subscriptions(atom()) -> list()). -show_subscriptions(Name) -> - gen_server:call(name(Name), show_subscriptions). - --spec(add_subscription(atom(), binary(), integer()) -> ok | {error, already_exists | validate_fail}). -add_subscription(Name, Topic, QoS) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {add_subscription, Topic, QoS}) - catch - _Error:_Reason -> - {error, validate_fail} - end. - --spec(del_subscription(atom(), binary()) -> ok | {error, validate_fail}). -del_subscription(Name, Topic) -> - try emqx_topic:validate({filter, Topic}) of - true -> - gen_server:call(name(Name), {del_subscription, Topic}) - catch - error:_Reason -> - {error, validate_fail} - end. - -status(Pid) -> - gen_server:call(Pid, status). - -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ - -init([Options]) -> - process_flag(trap_exit, true), - case get_value(start_type, Options, manual) of - manual -> ok; - auto -> erlang:send_after(1000, self(), start) - end, - ReconnectInterval = get_value(reconnect_interval, Options, 30000), - Mountpoint = format_mountpoint(get_value(mountpoint, Options)), - QueueOptions = get_value(queue, Options), - {ok, #state{mountpoint = Mountpoint, - queue_option = QueueOptions, - readq = [], - writeq = [], - options = Options, - reconnect_interval = ReconnectInterval}}. - -handle_call(start_bridge, _From, State = #state{client_pid = undefined}) -> - {Msg, NewState} = bridge(start, State), - {reply, #{msg => Msg}, NewState}; - -handle_call(start_bridge, _From, State) -> - {reply, #{msg => <<"bridge already started">>}, State}; - -handle_call(stop_bridge, _From, State = #state{client_pid = undefined}) -> - {reply, #{msg => <<"bridge not started">>}, State}; - -handle_call(stop_bridge, _From, State = #state{client_pid = Pid}) -> - emqx_client:disconnect(Pid), - {reply, #{msg => <<"stop bridge successfully">>}, State}; - -handle_call(status, _From, State = #state{client_pid = undefined}) -> - {reply, #{status => <<"Stopped">>}, State}; -handle_call(status, _From, State = #state{client_pid = _Pid})-> - {reply, #{status => <<"Running">>}, State}; - -handle_call(show_forwards, _From, State = #state{forwards = Forwards}) -> - {reply, Forwards, State}; - -handle_call({add_forward, Topic}, _From, State = #state{forwards = Forwards}) -> - case not lists:member(Topic, Forwards) of - true -> - emqx_broker:subscribe(Topic), - {reply, ok, State#state{forwards = [Topic | Forwards]}}; - false -> - {reply, {error, already_exists}, State} - end; - -handle_call({del_forward, Topic}, _From, State = #state{forwards = Forwards}) -> - case lists:member(Topic, Forwards) of - true -> - emqx_broker:unsubscribe(Topic), - {reply, ok, State#state{forwards = lists:delete(Topic, Forwards)}}; - false -> - {reply, ok, State} - end; - -handle_call(show_subscriptions, _From, State = #state{subscriptions = Subscriptions}) -> - {reply, Subscriptions, State}; - -handle_call({add_subscription, Topic, Qos}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> - case not lists:keymember(Topic, 1, Subscriptions) of - true -> - emqx_client:subscribe(ClientPid, {Topic, Qos}), - {reply, ok, State#state{subscriptions = [{Topic, Qos} | Subscriptions]}}; - false -> - {reply, {error, already_exists}, State} - end; - -handle_call({del_subscription, Topic}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> - case lists:keymember(Topic, 1, Subscriptions) of - true -> - emqx_client:unsubscribe(ClientPid, Topic), - {reply, ok, State#state{subscriptions = lists:keydelete(Topic, 1, Subscriptions)}}; - false -> - {reply, ok, State} - end; - -handle_call(Req, _From, State) -> - emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), - {noreply, State}. - -%%---------------------------------------------------------------- -%% Start or restart bridge -%%---------------------------------------------------------------- -handle_info(start, State) -> - {_Msg, NewState} = bridge(start, State), - {noreply, NewState}; - -handle_info(restart, State) -> - {_Msg, NewState} = bridge(restart, State), - {noreply, NewState}; - -%%---------------------------------------------------------------- -%% pop message from replayq and publish again -%%---------------------------------------------------------------- -handle_info(pop, State = #state{writeq = WriteQ, replayq = ReplayQ, - queue_option = #{batch_size := BatchSize}}) -> - {NewReplayQ, AckRef, NewReadQ} = replayq:pop(ReplayQ, #{count_limit => BatchSize}), - {NewReadQ1, NewWriteQ} = case NewReadQ of - [] -> {WriteQ, []}; - _ -> {NewReadQ, WriteQ} - end, - self() ! replay, - {noreply, State#state{readq = NewReadQ1, writeq = NewWriteQ, replayq = NewReplayQ, ackref = AckRef}}; - -handle_info(dump, State = #state{writeq = WriteQ, replayq = ReplayQ}) -> - NewReplayQueue = replayq:append(ReplayQ, lists:reverse(WriteQ)), - {noreply, State#state{replayq = NewReplayQueue, writeq = []}}; - -%%---------------------------------------------------------------- -%% replay message from replayq -%%---------------------------------------------------------------- -handle_info(replay, State = #state{client_pid = ClientPid, readq = ReadQ}) -> - {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []), - {noreply, State#state{readq = NewReadQ}}; - -%%---------------------------------------------------------------- -%% received local node message -%%---------------------------------------------------------------- -handle_info({dispatch, _, #message{topic = Topic, qos = QoS, payload = Payload, flags = #{retain := Retain}}}, - State = #state{client_pid = undefined, - mountpoint = Mountpoint}) - when QoS =< 1 -> - Msg = #mqtt_msg{qos = 1, - retain = Retain, - topic = mountpoint(Mountpoint, Topic), - payload = Payload}, - {noreply, en_writeq({undefined, Msg}, State)}; -handle_info({dispatch, _, #message{topic = Topic, qos = QoS ,payload = Payload, flags = #{retain := Retain}}}, - State = #state{client_pid = Pid, - mountpoint = Mountpoint}) - when QoS =< 1 -> - Msg = #mqtt_msg{qos = 1, - retain = Retain, - topic = mountpoint(Mountpoint, Topic), - payload = Payload}, - case emqx_client:publish(Pid, Msg) of - {ok, PktId} -> - {noreply, en_writeq({PktId, Msg}, State)}; - {error, {PktId, Reason}} -> - emqx_logger:error("[Bridge] Publish fail:~p", [Reason]), - {noreply, en_writeq({PktId, Msg}, State)} - end; - -%%---------------------------------------------------------------- -%% received remote node message -%%---------------------------------------------------------------- -handle_info({publish, #{qos := QoS, dup := Dup, retain := Retain, topic := Topic, - properties := Props, payload := Payload}}, State) -> - NewMsg0 = emqx_message:make(bridge, QoS, Topic, Payload), - NewMsg1 = emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, NewMsg0)), - emqx_broker:publish(NewMsg1), - {noreply, State}; - -%%---------------------------------------------------------------- -%% received remote puback message -%%---------------------------------------------------------------- -handle_info({puback, #{packet_id := PktId}}, State) -> - {noreply, delete(PktId, State)}; - -handle_info({'EXIT', Pid, normal}, State = #state{client_pid = Pid}) -> - emqx_logger:warning("[Bridge] stop ~p", [normal]), - self() ! dump, - {noreply, State#state{client_pid = undefined}}; - -handle_info({'EXIT', Pid, Reason}, State = #state{client_pid = Pid, - reconnect_interval = ReconnectInterval}) -> - emqx_logger:error("[Bridge] stop ~p", [Reason]), - self() ! dump, - erlang:send_after(ReconnectInterval, self(), restart), - {noreply, State#state{client_pid = undefined}}; - -handle_info(Info, State) -> - emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{}) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -subscribe_remote_topics(ClientPid, Subscriptions) -> - [begin emqx_client:subscribe(ClientPid, {bin(Topic), Qos}), {bin(Topic), Qos} end - || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})]. - -subscribe_local_topics(Options) -> - Topics = get_value(forwards, Options, []), - Subid = get_value(client_id, Options, <<"bridge">>), - [begin emqx_broker:subscribe(bin(Topic), #{qos => 1, subid => Subid}), bin(Topic) end - || Topic <- Topics, emqx_topic:validate({filter, bin(Topic)})]. - -proto_ver(mqttv3) -> v3; -proto_ver(mqttv4) -> v4; -proto_ver(mqttv5) -> v5. -address(Address) -> - case string:tokens(Address, ":") of - [Host] -> {Host, 1883}; - [Host, Port] -> {Host, list_to_integer(Port)} - end. -options(Options) -> - options(Options, []). -options([], Acc) -> - Acc; -options([{username, Username}| Options], Acc) -> - options(Options, [{username, Username}|Acc]); -options([{proto_ver, ProtoVer}| Options], Acc) -> - options(Options, [{proto_ver, proto_ver(ProtoVer)}|Acc]); -options([{password, Password}| Options], Acc) -> - options(Options, [{password, Password}|Acc]); -options([{keepalive, Keepalive}| Options], Acc) -> - options(Options, [{keepalive, Keepalive}|Acc]); -options([{client_id, ClientId}| Options], Acc) -> - options(Options, [{client_id, ClientId}|Acc]); -options([{clean_start, CleanStart}| Options], Acc) -> - options(Options, [{clean_start, CleanStart}|Acc]); -options([{address, Address}| Options], Acc) -> - {Host, Port} = address(Address), - options(Options, [{host, Host}, {port, Port}|Acc]); -options([{ssl, Ssl}| Options], Acc) -> - options(Options, [{ssl, Ssl}|Acc]); -options([{ssl_opts, SslOpts}| Options], Acc) -> - options(Options, [{ssl_opts, SslOpts}|Acc]); -options([_Option | Options], Acc) -> - options(Options, Acc). - -name(Id) -> - list_to_atom(lists:concat([?MODULE, "_", Id])). - -bin(L) -> iolist_to_binary(L). - -mountpoint(undefined, Topic) -> - Topic; -mountpoint(Prefix, Topic) -> - <>. - -format_mountpoint(undefined) -> - undefined; -format_mountpoint(Prefix) -> - binary:replace(bin(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). - -en_writeq(Msg, State = #state{replayq = ReplayQ, - queue_option = #{mem_cache := false}}) -> - NewReplayQ = replayq:append(ReplayQ, [Msg]), - State#state{replayq = NewReplayQ}; -en_writeq(Msg, State = #state{writeq = WriteQ, - queue_option = #{batch_size := BatchSize, - mem_cache := true}}) - when length(WriteQ) < BatchSize-> - State#state{writeq = [Msg | WriteQ]} ; -en_writeq(Msg, State = #state{writeq = WriteQ, replayq = ReplayQ, - queue_option = #{mem_cache := true}}) -> - NewReplayQ =replayq:append(ReplayQ, lists:reverse(WriteQ)), - State#state{writeq = [Msg], replayq = NewReplayQ}. - -publish_readq_msg(_ClientPid, [], NewReadQ) -> - {ok, NewReadQ}; -publish_readq_msg(ClientPid, [{_PktId, Msg} | ReadQ], NewReadQ) -> - {ok, PktId} = emqx_client:publish(ClientPid, Msg), - publish_readq_msg(ClientPid, ReadQ, [{PktId, Msg} | NewReadQ]). - -delete(PktId, State = #state{ replayq = ReplayQ, - readq = [], - queue_option = #{ mem_cache := false}}) -> - {NewReplayQ, NewAckRef, Msgs} = replayq:pop(ReplayQ, #{count_limit => 1}), - logger:debug("[Msg] PacketId ~p, Msg: ~p", [PktId, Msgs]), - ok = replayq:ack(NewReplayQ, NewAckRef), - case Msgs of - [{PktId, _Msg}] -> - self() ! pop, - State#state{ replayq = NewReplayQ, ackref = NewAckRef }; - [{_PktId, _Msg}] -> - NewReplayQ1 = replayq:append(NewReplayQ, Msgs), - self() ! pop, - State#state{ replayq = NewReplayQ1, ackref = NewAckRef }; - _Empty -> - State#state{ replayq = NewReplayQ, ackref = NewAckRef} - end; -delete(_PktId, State = #state{readq = [], writeq = [], replayq = ReplayQ, ackref = AckRef}) -> - ok = replayq:ack(ReplayQ, AckRef), - self() ! pop, - State; - -delete(PktId, State = #state{readq = [], writeq = WriteQ}) -> - State#state{writeq = lists:keydelete(PktId, 1, WriteQ)}; - -delete(PktId, State = #state{readq = ReadQ, replayq = ReplayQ, ackref = AckRef}) -> - NewReadQ = lists:keydelete(PktId, 1, ReadQ), - case NewReadQ of - [] -> - ok = replayq:ack(ReplayQ, AckRef), - self() ! pop; - _NewReadQ -> - ok - end, - State#state{ readq = NewReadQ }. - -bridge(Action, State = #state{options = Options, - replayq = ReplayQ, - queue_option - = QueueOption - = #{batch_size := BatchSize}}) - when BatchSize > 0 -> - case emqx_client:start_link([{owner, self()} | options(Options)]) of - {ok, ClientPid} -> - case emqx_client:connect(ClientPid) of - {ok, _} -> - emqx_logger:info("[Bridge] connected to remote successfully"), - Subs = subscribe_remote_topics(ClientPid, get_value(subscriptions, Options, [])), - Forwards = subscribe_local_topics(Options), - {NewReplayQ, AckRef, ReadQ} = open_replayq(ReplayQ, QueueOption), - {ok, NewReadQ} = publish_readq_msg(ClientPid, ReadQ, []), - {<<"start bridge successfully">>, - State#state{client_pid = ClientPid, - subscriptions = Subs, - readq = NewReadQ, - replayq = NewReplayQ, - ackref = AckRef, - forwards = Forwards}}; - {error, Reason} -> - emqx_logger:error("[Bridge] connect to remote failed! error: ~p", [Reason]), - {<<"connect to remote failed">>, - State#state{client_pid = ClientPid}} - end; - {error, Reason} -> - emqx_logger:error("[Bridge] ~p failed! error: ~p", [Action, Reason]), - {<<"start bridge failed">>, State} - end; -bridge(Action, State) -> - emqx_logger:error("[Bridge] ~p failed! error: batch_size should greater than zero", [Action]), - {<<"Open Replayq failed">>, State}. - -open_replayq(undefined, #{batch_size := BatchSize, - replayq_dir := ReplayqDir, - replayq_seg_bytes := ReplayqSegBytes}) -> - ReplayQ = replayq:open(#{dir => ReplayqDir, - seg_bytes => ReplayqSegBytes, - sizer => fun(Term) -> - size(term_to_binary(Term)) - end, - marshaller => fun({PktId, Msg}) -> - term_to_binary({PktId, Msg}); - (Bin) -> - binary_to_term(Bin) - end}), - replayq:pop(ReplayQ, #{count_limit => BatchSize}); -open_replayq(ReplayQ, #{batch_size := BatchSize}) -> - replayq:pop(ReplayQ, #{count_limit => BatchSize}). diff --git a/src/emqx_bridge_sup.erl b/src/emqx_bridge_sup.erl deleted file mode 100644 index baa857074..000000000 --- a/src/emqx_bridge_sup.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_bridge_sup). - --behavior(supervisor). - --include("emqx.hrl"). - --export([start_link/0, bridges/0]). - -%% Supervisor callbacks --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc List all bridges --spec(bridges() -> [{node(), map()}]). -bridges() -> - [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?MODULE)]. - -init([]) -> - BridgesOpts = emqx_config:get_env(bridges, []), - Bridges = [spec(Opts)|| Opts <- BridgesOpts], - {ok, {{one_for_one, 10, 100}, Bridges}}. - -spec({Id, Options})-> - #{id => Id, - start => {emqx_bridge, start_link, [Id, Options]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_bridge]}. diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 7ebd40769..0153f6570 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -18,6 +18,7 @@ -include("types.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_client.hrl"). -export([start_link/0, start_link/1]). -export([request/5, request/6, request_async/7, receive_response/3]). @@ -42,7 +43,7 @@ -export_type([client/0, properties/0, payload/0, pubopt/0, subopt/0, request_input/0, response_payload/0, request_handler/0, - corr_data/0]). + corr_data/0, mqtt_msg/0]). -export_type([host/0, option/0]). @@ -97,9 +98,6 @@ | {force_ping, boolean()} | {properties, properties()}). --record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, - packet_id, topic, props, payload}). - -type(mqtt_msg() :: #mqtt_msg{}). -record(state, {name :: atom(), diff --git a/src/emqx_local_bridge.erl b/src/emqx_local_bridge.erl deleted file mode 100644 index 0521e6d3f..000000000 --- a/src/emqx_local_bridge.erl +++ /dev/null @@ -1,157 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_local_bridge). - --behaviour(gen_server). - --include("emqx.hrl"). --include("emqx_mqtt.hrl"). - --export([start_link/5]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --define(PING_DOWN_INTERVAL, 1000). - --record(state, {pool, id, - node, subtopic, - qos = ?QOS_0, - topic_suffix = <<>>, - topic_prefix = <<>>, - mqueue :: emqx_mqueue:mqueue(), - max_queue_len = 10000, - ping_down_interval = ?PING_DOWN_INTERVAL, - status = up}). - --type(option() :: {qos, emqx_mqtt_types:qos()} | - {topic_suffix, binary()} | - {topic_prefix, binary()} | - {max_queue_len, pos_integer()} | - {ping_down_interval, pos_integer()}). - --export_type([option/0]). - -%% @doc Start a bridge --spec(start_link(term(), pos_integer(), atom(), binary(), [option()]) - -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id, Node, Topic, Options) -> - gen_server:start_link(?MODULE, [Pool, Id, Node, Topic, Options], [{hibernate_after, 5000}]). - -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ - -init([Pool, Id, Node, Topic, Options]) -> - process_flag(trap_exit, true), - true = gproc_pool:connect_worker(Pool, {Pool, Id}), - case net_kernel:connect_node(Node) of - true -> - true = erlang:monitor_node(Node, true), - Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), - emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}), - State = parse_opts(Options, #state{node = Node, subtopic = Topic}), - MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len, - store_qos0 => true}), - {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}}; - false -> - {stop, {cannot_connect_node, Node}} - end. - -parse_opts([], State) -> - State; -parse_opts([{qos, QoS} | Opts], State) -> - parse_opts(Opts, State#state{qos = QoS}); -parse_opts([{topic_suffix, Suffix} | Opts], State) -> - parse_opts(Opts, State#state{topic_suffix= Suffix}); -parse_opts([{topic_prefix, Prefix} | Opts], State) -> - parse_opts(Opts, State#state{topic_prefix = Prefix}); -parse_opts([{max_queue_len, Len} | Opts], State) -> - parse_opts(Opts, State#state{max_queue_len = Len}); -parse_opts([{ping_down_interval, Interval} | Opts], State) -> - parse_opts(Opts, State#state{ping_down_interval = Interval}); -parse_opts([_Opt | Opts], State) -> - parse_opts(Opts, State). - -handle_call(Req, _From, State) -> - emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({dispatch, _Topic, Msg}, State = #state{mqueue = Q, status = down}) -> - %% TODO: how to drop??? - {_Dropped, NewQ} = emqx_mqueue:in(Msg, Q), - {noreply, State#state{mqueue = NewQ}}; - -handle_info({dispatch, _Topic, Msg}, State = #state{node = Node, status = up}) -> - emqx_rpc:cast(Node, emqx_broker, publish, [transform(Msg, State)]), - {noreply, State}; - -handle_info({nodedown, Node}, State = #state{node = Node, ping_down_interval = Interval}) -> - emqx_logger:warning("[Bridge] node down: ~s", [Node]), - erlang:send_after(Interval, self(), ping_down_node), - {noreply, State#state{status = down}, hibernate}; - -handle_info({nodeup, Node}, State = #state{node = Node}) -> - %% TODO: Really fast?? - case emqx:is_running(Node) of - true -> emqx_logger:warning("[Bridge] Node up: ~s", [Node]), - {noreply, dequeue(State#state{status = up})}; - false -> self() ! {nodedown, Node}, - {noreply, State#state{status = down}} - end; - -handle_info(ping_down_node, State = #state{node = Node, ping_down_interval = Interval}) -> - Self = self(), - spawn_link(fun() -> - case net_kernel:connect_node(Node) of - true -> Self ! {nodeup, Node}; - false -> erlang:send_after(Interval, Self, ping_down_node) - end - end), - {noreply, State}; - -handle_info({'EXIT', _Pid, normal}, State) -> - {noreply, State}; - -handle_info(Info, State) -> - emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - gproc_pool:disconnect_worker(Pool, {Pool, Id}). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -dequeue(State = #state{mqueue = MQ}) -> - case emqx_mqueue:out(MQ) of - {empty, MQ1} -> - State#state{mqueue = MQ1}; - {{value, Msg}, MQ1} -> - handle_info({dispatch, Msg#message.topic, Msg}, State), - dequeue(State#state{mqueue = MQ1}) - end. - -transform(Msg = #message{topic = Topic}, #state{topic_prefix = Prefix, topic_suffix = Suffix}) -> - Msg#message{topic = <>}. - diff --git a/src/emqx_local_bridge_sup_sup.erl b/src/emqx_local_bridge_sup_sup.erl deleted file mode 100644 index 8a61d5936..000000000 --- a/src/emqx_local_bridge_sup_sup.erl +++ /dev/null @@ -1,74 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_local_bridge_sup_sup). - --behavior(supervisor). - --include("emqx.hrl"). - --export([start_link/0, bridges/0]). --export([start_bridge/2, start_bridge/3, stop_bridge/2]). - -%% Supervisor callbacks --export([init/1]). - --define(CHILD_ID(Node, Topic), {bridge_sup, Node, Topic}). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc List all bridges --spec(bridges() -> [{node(), emqx_topic:topic(), pid()}]). -bridges() -> - [{Node, Topic, Pid} || {?CHILD_ID(Node, Topic), Pid, supervisor, _} - <- supervisor:which_children(?MODULE)]. - -%% @doc Start a bridge --spec(start_bridge(node(), emqx_topic:topic()) -> {ok, pid()} | {error, term()}). -start_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> - start_bridge(Node, Topic, []). - --spec(start_bridge(node(), emqx_topic:topic(), [emqx_bridge:option()]) - -> {ok, pid()} | {error, term()}). -start_bridge(Node, _Topic, _Options) when Node =:= node() -> - {error, bridge_to_self}; -start_bridge(Node, Topic, Options) when is_atom(Node), is_binary(Topic) -> - Options1 = emqx_misc:merge_opts(emqx_config:get_env(bridge, []), Options), - supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)). - -%% @doc Stop a bridge --spec(stop_bridge(node(), emqx_topic:topic()) -> ok | {error, term()}). -stop_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> - ChildId = ?CHILD_ID(Node, Topic), - case supervisor:terminate_child(?MODULE, ChildId) of - ok -> supervisor:delete_child(?MODULE, ChildId); - Error -> Error - end. - -%%------------------------------------------------------------------------------ -%% Supervisor callbacks -%%------------------------------------------------------------------------------ - -init([]) -> - {ok, {{one_for_one, 10, 3600}, []}}. - -bridge_spec(Node, Topic, Options) -> - #{id => ?CHILD_ID(Node, Topic), - start => {emqx_local_bridge_sup, start_link, [Node, Topic, Options]}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [emqx_local_bridge_sup]}. - diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl index 20a08ba9f..016ff007f 100644 --- a/src/emqx_mqueue.erl +++ b/src/emqx_mqueue.erl @@ -67,7 +67,7 @@ default_priority => highest | lowest, store_qos0 => boolean() }). --type(message() :: pemqx_types:message()). +-type(message() :: emqx_types:message()). -type(stat() :: {len, non_neg_integer()} | {max_len, non_neg_integer()} diff --git a/src/emqx_portal_connect.erl b/src/emqx_portal_connect.erl new file mode 100644 index 000000000..35f4b53dc --- /dev/null +++ b/src/emqx_portal_connect.erl @@ -0,0 +1,65 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_connect). + +-export([start/2]). + +-export_type([config/0, connection/0]). + +-optional_callbacks([]). + +-type config() :: map(). +-type connection() :: term(). +-type conn_ref() :: term(). +-type batch() :: emqx_protal:batch(). +-type batch_ref() :: reference(). + +-include("logger.hrl"). + +%% establish the connection to remote node/cluster +%% protal worker (the caller process) should be expecting +%% a message {disconnected, conn_ref()} when disconnected. +-callback start(config()) -> {ok, conn_ref(), connection()} | {error, any()}. + +%% send to remote node/cluster +%% portal worker (the caller process) should be expecting +%% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster +-callback send(connection(), batch()) -> {ok, batch_ref()} | {error, any()}. + +%% called when owner is shutting down. +-callback stop(conn_ref(), connection()) -> ok. + +start(Module, Config) -> + case Module:start(Config) of + {ok, Ref, Conn} -> + {ok, Ref, Conn}; + {error, Reason} -> + Config1 = obfuscate(Config), + ?ERROR("Failed to connect with module=~p\n" + "config=~p\nreason:~p", [Module, Config1, Reason]), + error + end. + +obfuscate(Map) -> + maps:fold(fun(K, V, Acc) -> + case is_sensitive(K) of + true -> [{K, '***'} | Acc]; + false -> [{K, V} | Acc] + end + end, [], Map). + +is_sensitive(passsword) -> true; +is_sensitive(_) -> false. + diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 5f62df904..0e6ebb08a 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -61,10 +61,7 @@ init([]) -> RouterSup = supervisor_spec(emqx_router_sup), %% Broker Sup BrokerSup = supervisor_spec(emqx_broker_sup), - %% BridgeSup - LocalBridgeSup = supervisor_spec(emqx_local_bridge_sup_sup), - - BridgeSup = supervisor_spec(emqx_bridge_sup), + PortalSup = supervisor_spec(emqx_portal_sup), %% AccessControl AccessControl = worker_spec(emqx_access_control), %% Session Manager @@ -77,8 +74,7 @@ init([]) -> [KernelSup, RouterSup, BrokerSup, - LocalBridgeSup, - BridgeSup, + PortalSup, AccessControl, SMSup, CMSup, diff --git a/src/emqx_topic.erl b/src/emqx_topic.erl index 59f592984..4c90c3f39 100644 --- a/src/emqx_topic.erl +++ b/src/emqx_topic.erl @@ -20,7 +20,7 @@ -export([triples/1]). -export([words/1]). -export([wildcard/1]). --export([join/1]). +-export([join/1, prepend/2]). -export([feed_var/3]). -export([systop/1]). -export([parse/1, parse/2]). @@ -129,6 +129,18 @@ join(root, W) -> join(Parent, W) -> <<(bin(Parent))/binary, $/, (bin(W))/binary>>. +%% @doc Prepend a topic prefix. +%% Ensured to have only one / between prefix and suffix. +prepend(root, W) -> bin(W); +prepend(undefined, W) -> bin(W); +prepend(<<>>, W) -> bin(W); +prepend(Parent0, W) -> + Parent = bin(Parent0), + case binary:last(Parent) of + $/ -> <>; + _ -> join(Parent, W) + end. + bin('') -> <<>>; bin('+') -> <<"+">>; bin('#') -> <<"#">>; diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl new file mode 100644 index 000000000..7e7d678f6 --- /dev/null +++ b/src/portal/emqx_portal.erl @@ -0,0 +1,356 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc Portal works in two layers (1) batching layer (2) transport layer +%% The `portal' batching layer collects local messages in batches and sends over +%% to remote MQTT node/cluster via `connetion' transport layer. +%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be +%% the `gen_rpc' based implementation `emqx_portal_rpc'. Otherwise `connection' +%% has to be `emqx_portal_mqtt'. +%% +%% ``` +%% +------+ +--------+ +%% | EMQX | | REMOTE | +%% | | | | +%% | (portal) <==(connection)==> | | +%% | | | | +%% | | | | +%% +------+ +--------+ +%% ''' +%% +%% +%% This module implements 2 kinds of APIs with regards to batching and +%% messaging protocol. (1) A `gen_statem' based local batch collector; +%% (2) APIs for incoming remote batches/messages. +%% +%% Batch collector state diagram +%% +%% [connecting] --(2)--> [connected] +%% | ^ | +%% | | | +%% '--(1)---'--------(3)------' +%% +%% (1): timeout +%% (2): successfuly connected to remote node/cluster +%% (3): received {disconnected, conn_ref(), Reason} OR +%% failed to send to remote node/cluster. +%% +%% NOTE: A portal worker may subscribe to multiple (including wildcard) +%% local topics, and the underlying `emqx_portal_connect' may subscribe to +%% multiple remote topics, however, worker/connections are not designed +%% to support automatic load-balancing, i.e. in case it can not keep up +%% with the amount of messages comming in, administrator should split and +%% balance topics between worker/connections manually. + +-module(emqx_portal). +-behaviour(gen_statem). + +%% APIs +-export([start_link/2, + import_batch/2, + handle_ack/2, + stop/1 + ]). + +%% gen_statem callbacks +-export([terminate/3, code_change/4, init/1, callback_mode/0]). + +%% state functions +-export([connecting/3, connected/3]). + +-export_type([config/0, + batch/0, + ref/0 + ]). + +-type config() :: map(). +-type batch() :: [emqx_portal_msg:msg()]. +-type ref() :: reference(). + +-include("logger.hrl"). +-include("emqx_mqtt.hrl"). + +-define(DEFAULT_BATCH_COUNT, 100). +-define(DEFAULT_BATCH_BYTES, 1 bsl 20). +-define(DEFAULT_SEND_AHEAD, 8). +-define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)). +-define(DEFAULT_SEG_BYTES, (1 bsl 20)). +-define(maybe_send, {next_event, internal, maybe_send}). + +%% @doc Start a portal worker. Supported configs: +%% connect_module: The module which implements emqx_portal_connect behaviour +%% and work as message batch transport layer +%% reconnect_delay_ms: Delay in milli-seconds for the portal worker to retry +%% in case of transportation failure. +%% max_inflight_batches: Max number of batches allowed to send-ahead before +%% receiving confirmation from remote node/cluster +%% mountpoint: The topic mount point for messages sent to remote node/cluster +%% `undefined', `<<>>' or `""' to disalble +%% forwards: Local topics to subscribe. +%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each +%% send call towards emqx_portal_connect +%% queue.batch_count_limit: Max number of messages to collect in a batch for +%% each send call towards eqmx_portal_connect +%% queue.replayq_dir: Directory where replayq should persist messages +%% queue.replayq_seg_bytes: Size in bytes for each replqyq segnment file +%% +%% Find more connection specific configs in the callback modules +%% of emqx_portal_connect behaviour. +start_link(Name, Config) when is_list(Config) -> + start_link(Name, maps:from_list(Config)); +start_link(Name, Config) -> + gen_statem:start_link({local, Name}, ?MODULE, Config, []). + +stop(Pid) -> gen_statem:stop(Pid). + +%% @doc This function is to be evaluated on message/batch receiver side. +-spec import_batch(batch(), fun(() -> ok)) -> ok. +import_batch(Batch, AckFun) -> + lists:foreach(fun emqx_broker:publish/1, emqx_portal_msg:to_broker_msgs(Batch)), + AckFun(). + +%% @doc This function is to be evaluated on message/batch exporter side +%% when message/batch is accepted by remote node. +-spec handle_ack(pid(), ref()) -> ok. +handle_ack(Pid, Ref) when node() =:= node(Pid) -> + Pid ! {batch_ack, Ref}, + ok. + + +callback_mode() -> [state_functions, state_enter]. + +%% @doc Config should be a map(). +init(Config) -> + erlang:process_flag(trap_exit, true), + Get = fun(K, D) -> maps:get(K, Config, D) end, + QCfg = maps:get(queue, Config, #{}), + GetQ = fun(K, D) -> maps:get(K, QCfg, D) end, + Dir = GetQ(replayq_dir, undefined), + QueueConfig = + case Dir =:= undefined orelse Dir =:= "" of + true -> #{mem_only => true}; + false -> #{dir => Dir, + seg_bytes => GetQ(replayq_seg_bytes, ?DEFAULT_SEG_BYTES) + } + end, + Queue = replayq:open(QueueConfig#{sizer => fun emqx_portal_msg:estimate_size/1, + marshaller => fun msg_marshaller/1}), + Topics = Get(forwards, []), + ok = subscribe_local_topics(Topics), + ConnectModule = maps:get(connect_module, Config), + ConnectConfig = maps:without([connect_module, + queue, + reconnect_delay_ms, + max_inflight_batches, + mountpoint, + forwards + ], Config), + ConnectFun = fun() -> emqx_portal_connect:start(ConnectModule, ConnectConfig) end, + {ok, connecting, + #{connect_module => ConnectModule, + connect_fun => ConnectFun, + reconnect_delay_ms => maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS), + batch_bytes_limit => GetQ(batch_bytes_limit, ?DEFAULT_BATCH_BYTES), + batch_count_limit => GetQ(batch_count_limit, ?DEFAULT_BATCH_COUNT), + max_inflight_batches => Get(max_inflight_batches, ?DEFAULT_SEND_AHEAD), + mountpoint => format_mountpoint(Get(mountpoint, undefined)), + topics => Topics, + replayq => Queue, + inflight => [] + }}. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +terminate(_Reason, _StateName, #{replayq := Q} = State) -> + _ = disconnect(State), + _ = replayq:close(Q), + ok. + +%% @doc Connecting state is a state with timeout. +%% After each timeout, it re-enters this state and start a retry until +%% successfuly connected to remote node/cluster. +connecting(enter, connected, #{reconnect_delay_ms := Timeout}) -> + Action = {state_timeout, Timeout, reconnect}, + {keep_state_and_data, Action}; +connecting(enter, connecting, #{reconnect_delay_ms := Timeout, + connect_fun := ConnectFun} = State) -> + case ConnectFun() of + {ok, ConnRef, Conn} -> + Action = {state_timeout, 0, connected}, + {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; + error -> + Action = {state_timeout, Timeout, reconnect}, + {keep_state_and_data, Action} + end; +connecting(state_timeout, connected, State) -> + {next_state, connected, State}; +connecting(state_timeout, reconnect, _State) -> + repeat_state_and_data; +connecting(info, {batch_ack, Ref}, State) -> + case do_ack(State, Ref) of + {ok, NewState} -> + {keep_state, NewState}; + _ -> + keep_state_and_data + end; +connecting(Type, Content, State) -> + common(connecting, Type, Content, State). + +%% @doc Send batches to remote node/cluster when in 'connected' state. +connected(enter, _OldState, #{inflight := Inflight} = State) -> + case retry_inflight(State#{inflight := []}, Inflight) of + {ok, NewState} -> + Action = {state_timeout, 0, success}, + {keep_state, NewState, Action}; + {error, NewState} -> + Action = {state_timeout, 0, failure}, + {keep_state, disconnect(NewState), Action} + end; +connected(state_timeout, failure, State) -> + {next_state, connecting, State}; +connected(state_timeout, success, State) -> + {keep_state, State, ?maybe_send}; +connected(internal, maybe_send, State) -> + case pop_and_send(State) of + {ok, NewState} -> + {keep_state, NewState}; + {error, NewState} -> + {next_state, connecting, disconnect(NewState)} + end; +connected(info, {disconnected, ConnRef, Reason}, + #{conn_ref := ConnRef, connection := Conn} = State) -> + ?INFO("Portal ~p diconnected~nreason=~p", [Conn, Reason]), + {next_state, connecting, + State#{conn_ref := undefined, + connection := undefined + }}; +connected(info, {batch_ack, Ref}, State) -> + case do_ack(State, Ref) of + stale -> + keep_state_and_data; + bad_order -> + %% try re-connect then re-send + {next_state, connecting, disconnect(State)}; + {ok, NewState} -> + {keep_state, NewState} + end; +connected(Type, Content, State) -> + common(connected, Type, Content, State). + +%% Common handlers +common(_StateName, info, {dispatch, _, Msg}, + #{replayq := Q} = State) -> + NewQ = replayq:append(Q, collect([Msg])), + {keep_state, State#{replayq => NewQ}, ?maybe_send}; +common(StateName, Type, Content, State) -> + ?INFO("Ignored unknown ~p event ~p at state ~p", [Type, Content, StateName]), + {keep_state, State}. + +collect(Acc) -> + receive + {dispatch, _, Msg} -> + collect([Msg | Acc]) + after + 0 -> + lists:reverse(Acc) + end. + +%% Retry all inflight (previously sent but not acked) batches. +retry_inflight(State, []) -> {ok, State}; +retry_inflight(#{inflight := Inflight} = State, + [#{q_ack_ref := QAckRef, batch := Batch} | T] = Remain) -> + case do_send(State, QAckRef, Batch) of + {ok, NewState} -> + retry_inflight(NewState, T); + {error, Reason} -> + ?ERROR("Inflight retry failed\n~p", [Reason]), + {error, State#{inflight := Inflight ++ Remain}} + end. + +pop_and_send(#{inflight := Inflight, + max_inflight_batches := Max + } = State) when length(Inflight) >= Max -> + {ok, State}; +pop_and_send(#{replayq := Q, + batch_count_limit := CountLimit, + batch_bytes_limit := BytesLimit + } = State) -> + case replayq:is_empty(Q) of + true -> + {ok, State}; + false -> + Opts = #{count_limit => CountLimit, bytes_limit => BytesLimit}, + {Q1, QAckRef, Batch} = replayq:pop(Q, Opts), + do_send(State#{replayq := Q1}, QAckRef, Batch) + end. + +%% Assert non-empty batch because we have a is_empty check earlier. +do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) -> + case maybe_send(State, Batch) of + {ok, Ref} -> + NewInflight = Inflight ++ [#{q_ack_ref => QAckRef, + send_ack_ref => Ref, + batch => Batch + }], + {ok, State#{inflight := NewInflight}}; + {error, Reason} -> + ?INFO("Batch produce failed\n~p", [Reason]), + {error, State} + end. + +do_ack(State = #{inflight := [#{send_ack_ref := Ref} | Rest]}, Ref) -> + {ok, State#{inflight := Rest}}; +do_ack(#{inflight := Inflight}, Ref) -> + case lists:any(fun(#{send_ack_ref := Ref0}) -> Ref0 =:= Ref end, Inflight) of + true -> bad_order; + false -> stale + end. + +subscribe_local_topics(Topics) -> + lists:foreach( + fun(Topic0) -> + Topic = iolist_to_binary(Topic0), + emqx_topic:validate({filter, Topic}) orelse erlang:error({bad_topic, Topic}), + emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}) + end, Topics). + +name() -> {_, Name} = process_info(self(), registered_name), Name. + +disconnect(#{connection := Conn, + conn_ref := ConnRef, + connect_module := Module + } = State) when Conn =/= undefined -> + ok = Module:stop(ConnRef, Conn), + State#{conn_ref => undefined, + connection => undefined + }; +disconnect(State) -> State. + +%% Called only when replayq needs to dump it to disk. +msg_marshaller(Bin) when is_binary(Bin) -> emqx_portal_msg:from_binary(Bin); +msg_marshaller(Msg) -> emqx_portal_msg:to_binary(Msg). + +%% Return {ok, SendAckRef} or {error, Reason} +maybe_send(#{connect_module := Module, + connection := Connection, + mountpoint := Mountpoint + }, Batch) -> + Module:send(Connection, [emqx_portal_msg:apply_mountpoint(M, Mountpoint) || M <- Batch]). + +format_mountpoint(undefined) -> + undefined; +format_mountpoint(Prefix) -> + binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). + diff --git a/src/portal/emqx_portal_msg.erl b/src/portal/emqx_portal_msg.erl new file mode 100644 index 000000000..af9cc1b01 --- /dev/null +++ b/src/portal/emqx_portal_msg.erl @@ -0,0 +1,61 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_msg). + +-export([ to_binary/1 + , from_binary/1 + , apply_mountpoint/2 + , to_broker_msgs/1 + , estimate_size/1 + ]). + +-export_type([msg/0]). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-type msg() :: emqx_types:message(). + +%% @doc Mount topic to a prefix. +-spec apply_mountpoint(msg(), undefined | binary()) -> msg(). +apply_mountpoint(#{topic := Topic} = Msg, Mountpoint) -> + Msg#{topic := topic(Mountpoint, Topic)}. + +%% @doc Make `binary()' in order to make iodata to be persisted on disk. +-spec to_binary(msg()) -> binary(). +to_binary(Msg) -> term_to_binary(Msg). + +%% @doc Unmarshal binary into `msg()'. +-spec from_binary(binary()) -> msg(). +from_binary(Bin) -> binary_to_term(Bin). + +%% @doc Estimate the size of a message. +%% Count only the topic length + payload size +-spec estimate_size(msg()) -> integer(). +estimate_size(#{topic := Topic, payload := Payload}) -> + size(Topic) + size(Payload). + +%% @doc By message/batch receiver, transform received batch into +%% messages to dispatch to local brokers. +to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch). + +to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, + properties := Props, payload := Payload}) -> + emqx_message:set_headers(Props, + emqx_message:set_flags(#{dup => Dup, retain => Retain}, + emqx_message:make(portal, QoS, Topic, Payload))). + +topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). + diff --git a/src/portal/emqx_portal_rpc.erl b/src/portal/emqx_portal_rpc.erl new file mode 100644 index 000000000..8b1136353 --- /dev/null +++ b/src/portal/emqx_portal_rpc.erl @@ -0,0 +1,106 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc This module implements EMQX Portal transport layer based on gen_rpc. + +-module(emqx_portal_rpc). +-behaviour(emqx_portal_connect). + +%% behaviour callbacks +-export([start/1, + send/2, + stop/2 + ]). + +%% Internal exports +-export([ handle_send/2 + , handle_ack/2 + , heartbeat/2 + ]). + +-type batch_ref() :: emqx_portal:batch_ref(). +-type batch() :: emqx_portal:batch(). + +-define(HEARTBEAT_INTERVAL, timer:seconds(1)). + +-define(RPC, gen_rpc). + +start(#{address := Remote}) -> + case poke(Remote) of + ok -> + Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), Remote]), + {ok, Pid, Remote}; + Error -> + Error + end. + +stop(Pid, _Remote) when is_pid(Pid) -> + Ref = erlang:monitor(process, Pid), + unlink(Pid), + Pid ! stop, + receive + {'DOWN', Ref, process, Pid, _Reason} -> + ok + after + 1000 -> + exit(Pid, kill) + end, + ok. + +%% @doc Callback for `emqx_portal_connect' behaviour +-spec send(node(), batch()) -> {ok, batch_ref()} | {error, any()}. +send(Remote, Batch) -> + Sender = self(), + case ?RPC:call(Remote, ?MODULE, handle_send, [Sender, Batch]) of + {ok, Ref} -> {ok, Ref}; + {badrpc, Reason} -> {error, Reason} + end. + +%% @doc Handle send on receiver side. +-spec handle_send(pid(), batch()) -> {ok, batch_ref()} | {error, any()}. +handle_send(SenderPid, Batch) -> + SenderNode = node(SenderPid), + Ref = make_ref(), + AckFun = fun() -> ?RPC:cast(SenderNode, ?MODULE, handle_ack, [SenderPid, Ref]), ok end, + case emqx_portal:import_batch(Batch, AckFun) of + ok -> {ok, Ref}; + Error -> Error + end. + +%% @doc Handle batch ack in sender node. +handle_ack(SenderPid, Ref) -> + ok = emqx_portal:handle_ack(SenderPid, Ref). + +%% @hidden Heartbeat loop +heartbeat(Parent, RemoteNode) -> + Interval = ?HEARTBEAT_INTERVAL, + receive + stop -> exit(normal) + after + Interval -> + case poke(RemoteNode) of + ok -> + ?MODULE:heartbeat(Parent, RemoteNode); + {error, Reason} -> + Parent ! {disconnected, self(), Reason}, + exit(normal) + end + end. + +poke(Node) -> + case ?RPC:call(Node, erlang, node, []) of + Node -> ok; + {badrpc, Reason} -> {error, Reason} + end. + diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl new file mode 100644 index 000000000..79afd6352 --- /dev/null +++ b/src/portal/emqx_portal_sup.erl @@ -0,0 +1,54 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_sup). +-behavior(supervisor). + +-export([start_link/0, start_link/1]). + +-export([init/1]). + +-define(SUP, ?MODULE). +-define(WORKER_SUP, emqx_portal_worker_sup). + +start_link() -> start_link(?SUP). + +start_link(Name) -> + supervisor:start_link({local, Name}, ?MODULE, Name). + +init(?SUP) -> + Sp = fun(Name) -> + #{id => Name, + start => {?MODULE, start_link, [Name]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + } + end, + {ok, {{one_for_one, 5, 10}, [Sp(?WORKER_SUP)]}}; +init(?WORKER_SUP) -> + BridgesConf = emqx_config:get_env(bridges, []), + BridgesSpec = lists:map(fun portal_spec/1, BridgesConf), + {ok, {{one_for_one, 10, 100}, BridgesSpec}}. + +portal_spec({Name, Config}) -> + #{id => Name, + start => {emqx_portal, start_link, [Name, Config]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_portal] + }. + diff --git a/test/emqx_bridge_SUITE.erl b/test/emqx_bridge_SUITE.erl deleted file mode 100644 index 9681c27e6..000000000 --- a/test/emqx_bridge_SUITE.erl +++ /dev/null @@ -1,58 +0,0 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - --module(emqx_bridge_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - [bridge_test]. - -init_per_suite(Config) -> - emqx_ct_broker_helpers:run_setup_steps(), - Config. - -end_per_suite(_Config) -> - emqx_ct_broker_helpers:run_teardown_steps(). - -bridge_test(_) -> - #{msg := <<"start bridge successfully">>} - = emqx_bridge:start_bridge(aws), - test_forwards(), - test_subscriptions(0), - test_subscriptions(1), - test_subscriptions(2), - #{msg := <<"stop bridge successfully">>} - = emqx_bridge:stop_bridge(aws), - ok. - -test_forwards() -> - emqx_bridge:add_forward(aws, <<"test_forwards">>), - [<<"test_forwards">>, <<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws), - emqx_bridge:del_forward(aws, <<"test_forwards">>), - [<<"topic1/#">>, <<"topic2/#">>] = emqx_bridge:show_forwards(aws), - ok. - -test_subscriptions(QoS) -> - emqx_bridge:add_subscription(aws, <<"test_subscriptions">>, QoS), - [{<<"test_subscriptions">>, QoS}, - {<<"cmd/topic1">>, 1}, - {<<"cmd/topic2">>, 1}] = emqx_bridge:show_subscriptions(aws), - emqx_bridge:del_subscription(aws, <<"test_subscriptions">>), - [{<<"cmd/topic1">>,1}, {<<"cmd/topic2">>,1}] = emqx_bridge:show_subscriptions(aws), - ok. diff --git a/test/emqx_ct_broker_helpers.erl b/test/emqx_ct_broker_helpers.erl index 1ab79e8a9..1ef5b1fa3 100644 --- a/test/emqx_ct_broker_helpers.erl +++ b/test/emqx_ct_broker_helpers.erl @@ -156,24 +156,30 @@ flush(Msgs) -> end. bridge_conf() -> - [{aws, - [{username,"user"}, - {address,"127.0.0.1:1883"}, - {clean_start,true}, - {client_id,"bridge_aws"}, - {forwards,["topic1/#","topic2/#"]}, - {keepalive,60000}, - {max_inflight,32}, - {mountpoint,"bridge/aws/${node}/"}, - {password,"passwd"}, - {proto_ver,mqttv4}, - {queue, - #{batch_size => 1000,mem_cache => true, - replayq_dir => "data/emqx_aws_bridge/", - replayq_seg_bytes => 10485760}}, - {reconnect_interval,30000}, - {retry_interval,20000}, - {ssl,false}, - {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]}, - {start_type,manual}, - {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}]. \ No newline at end of file + [ {local_rpc, + [{connect_module, emqx_portal_rpc}, + {address, node()}, + {forwards, ["portal-1/#", "portal-2/#"]} + ]} + ]. + % [{aws, + % [{connect_module, emqx_portal_mqtt}, + % {username,"user"}, + % {address,"127.0.0.1:1883"}, + % {clean_start,true}, + % {client_id,"bridge_aws"}, + % {forwards,["topic1/#","topic2/#"]}, + % {keepalive,60000}, + % {max_inflight,32}, + % {mountpoint,"bridge/aws/${node}/"}, + % {password,"passwd"}, + % {proto_ver,mqttv4}, + % {queue, + % #{batch_coun t_limit => 1000, + % replayq_dir => "data/emqx_aws_bridge/", + % replayq_seg_bytes => 10485760}}, + % {reconnect_delay_ms,30000}, + % {ssl,false}, + % {ssl_opts,[{versions,[tlsv1,'tlsv1.1','tlsv1.2']}]}, + % {start_type,manual}, + % {subscriptions,[{"cmd/topic1",1},{"cmd/topic2",1}]}]}]. diff --git a/test/emqx_pool_SUITE.erl b/test/emqx_pool_SUITE.erl index ea648709f..d2fb90afc 100644 --- a/test/emqx_pool_SUITE.erl +++ b/test/emqx_pool_SUITE.erl @@ -62,7 +62,7 @@ async_submit_mfa(_Config) -> emqx_pool:async_submit(fun ?MODULE:test_mfa/0, []). async_submit_crash(_) -> - emqx_pool:async_submit(fun() -> A = 1, A = 0 end). + emqx_pool:async_submit(fun() -> error(test) end). t_unexpected(_) -> Pid = emqx_pool:worker(), diff --git a/test/emqx_portal_rpc_tests.erl b/test/emqx_portal_rpc_tests.erl new file mode 100644 index 000000000..5fd7608c0 --- /dev/null +++ b/test/emqx_portal_rpc_tests.erl @@ -0,0 +1,43 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_rpc_tests). +-include_lib("eunit/include/eunit.hrl"). + +send_and_ack_test() -> + %% delegate from gen_rpc to rpc for unit test + meck:new(gen_rpc, [passthrough, no_history]), + meck:expect(gen_rpc, call, 4, + fun(Node, Module, Fun, Args) -> + rpc:call(Node, Module, Fun, Args) + end), + meck:expect(gen_rpc, cast, 4, + fun(Node, Module, Fun, Args) -> + rpc:cast(Node, Module, Fun, Args) + end), + meck:new(emqx_portal, [passthrough, no_history]), + meck:expect(emqx_portal, import_batch, 2, + fun(batch, AckFun) -> AckFun() end), + try + {ok, Pid, Node} = emqx_portal_rpc:start(#{address => node()}), + {ok, Ref} = emqx_portal_rpc:send(Node, batch), + receive + {batch_ack, Ref} -> + ok + end, + ok = emqx_portal_rpc:stop(Pid, Node) + after + meck:unload(gen_rpc), + meck:unload(emqx_portal) + end. diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl new file mode 100644 index 000000000..5826a327e --- /dev/null +++ b/test/emqx_portal_tests.erl @@ -0,0 +1,146 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_tests). +-behaviour(emqx_portal_connect). + +-include_lib("eunit/include/eunit.hrl"). +-include("emqx_mqtt.hrl"). + +-define(PORTAL_NAME, test_portal). +-define(WAIT(PATTERN, TIMEOUT), + receive + PATTERN -> + ok + after + TIMEOUT -> + error(timeout) + end). + +%% stub callbacks +-export([start/1, send/2, stop/2]). + +start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) -> + case is_pid(Pid) of + true -> Pid ! {connection_start_attempt, Ref}; + false -> ok + end, + Result. + +send(SendFun, Batch) when is_function(SendFun, 1) -> + SendFun(Batch). + +stop(_Ref, _Pid) -> ok. + +%% portal worker should retry connecting remote node indefinitely +reconnect_test() -> + Ref = make_ref(), + Config = make_config(Ref, self(), {error, test}), + {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), + %% assert name registered + ?assertEqual(Pid, whereis(?PORTAL_NAME)), + ?WAIT({connection_start_attempt, Ref}, 1000), + %% expect same message again + ?WAIT({connection_start_attempt, Ref}, 1000), + ok = emqx_portal:stop(?PORTAL_NAME), + ok. + +%% connect first, disconnect, then connect again +disturbance_test() -> + Ref = make_ref(), + Config = make_config(Ref, self(), {ok, Ref, connection}), + {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), + ?assertEqual(Pid, whereis(?PORTAL_NAME)), + ?WAIT({connection_start_attempt, Ref}, 1000), + Pid ! {disconnected, Ref, test}, + ?WAIT({connection_start_attempt, Ref}, 1000), + ok = emqx_portal:stop(?PORTAL_NAME). + +%% buffer should continue taking in messages when disconnected +buffer_when_disconnected_test_() -> + {timeout, 5000, fun test_buffer_when_disconnected/0}. + +test_buffer_when_disconnected() -> + Ref = make_ref(), + Nums = lists:seq(1, 100), + Sender = spawn_link(fun() -> receive {portal, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end), + SenderMref = monitor(process, Sender), + Receiver = spawn_link(fun() -> receive {portal, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end), + ReceiverMref = monitor(process, Receiver), + SendFun = fun(Batch) -> + BatchRef = make_ref(), + Receiver ! {batch, BatchRef, Batch}, + {ok, BatchRef} + end, + Config0 = make_config(Ref, false, {ok, Ref, SendFun}), + Config = Config0#{reconnect_delay_ms => 100}, + {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), + Sender ! {portal, Pid}, + Receiver ! {portal, Pid}, + ?assertEqual(Pid, whereis(?PORTAL_NAME)), + Pid ! {disconnected, Ref, test}, + ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 2000), + ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), + ok = emqx_portal:stop(?PORTAL_NAME). + +%% Feed messages to portal +sender_loop(_Pid, [], _) -> exit(normal); +sender_loop(Pid, [Num | Rest], Interval) -> + random_sleep(Interval), + Pid ! {dispatch, dummy, make_msg(Num)}, + sender_loop(Pid, Rest, Interval). + +%% Feed acknowledgments to portal +receiver_loop(_Pid, [], _) -> ok; +receiver_loop(Pid, Nums, Interval) -> + receive + {batch, BatchRef, Batch} -> + Rest = match_nums(Batch, Nums), + random_sleep(Interval), + emqx_portal:handle_ack(Pid, BatchRef), + receiver_loop(Pid, Rest, Interval) + end. + +random_sleep(MaxInterval) -> + case rand:uniform(MaxInterval) - 1 of + 0 -> ok; + T -> timer:sleep(T) + end. + +match_nums([], Rest) -> Rest; +match_nums([#{payload := P} | Rest], Nums) -> + I = binary_to_integer(P), + case Nums of + [I | NumsLeft] -> match_nums(Rest, NumsLeft); + _ -> error({I, Nums}) + end. + +make_config(Ref, TestPid, Result) -> + #{test_pid => TestPid, + test_ref => Ref, + connect_module => ?MODULE, + reconnect_delay_ms => 50, + connect_result => Result + }. + +make_msg(I) -> + Payload = integer_to_binary(I), + #{qos => ?QOS_1, + dup => false, + retain => false, + topic => <<"test/topic">>, + properties => [], + payload => Payload + }. + From 141af0d02cd220ea9c9196b0e18b2feaf3e02fb7 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Thu, 7 Feb 2019 04:58:55 +0100 Subject: [PATCH 02/26] Add message handler callbacks option to emqx_client In this commit, msg_handler option is added to emqx_client. so the caller can provide callbacks to handle puback, publish, as well as disconnected events instead of always delivered as message like Owner ! {publish, Msg} to the owner process. This is to make it ready to implement emqx_portal_connect on top of emqx_client. --- src/emqx_client.erl | 96 ++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 0153f6570..389c9e902 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -59,7 +59,7 @@ -define(RESPONSE_TIMEOUT_SECONDS, timer:seconds(5)). --define(NO_HANDLER, undefined). +-define(NO_REQ_HANDLER, undefined). -define(NO_GROUP, <<>>). @@ -67,10 +67,23 @@ -type(host() :: inet:ip_address() | inet:hostname()). --type corr_data() :: binary(). +-type(corr_data() :: binary()). + +%% NOTE: Message handler is different from request handler. +%% Message handler is a set of callbacks defined to handle MQTT messages as well as +%% the disconnect event. +%% Request handler is a callback to handle received MQTT message as in 'request', +%% and publish another MQTT message back to the defined topic as in 'response'. +%% `owner' and `msg_handler' has no effect when `request_handler' is set. +-define(NO_MSG_HDLR, undefined). +-type(msg_handler() :: #{puback := fun((_) -> any()), + publish := fun((emqx_types:message()) -> any()), + disconnected := fun(({reason_code(), _Properties :: term()}) -> any()) + }). -type(option() :: {name, atom()} | {owner, pid()} + | {msg_handler, msg_handler()} | {host, host()} | {hosts, [{host(), inet:port_number()}]} | {port, inet:port_number()} @@ -102,6 +115,7 @@ -record(state, {name :: atom(), owner :: pid(), + msg_handler :: ?NO_MSG_HDLR | msg_handler(), host :: host(), port :: inet:port_number(), hosts :: [{host(), inet:port_number()}], @@ -497,7 +511,7 @@ init([Options]) -> auto_ack = true, ack_timeout = ?DEFAULT_ACK_TIMEOUT, retry_interval = 0, - request_handler = ?NO_HANDLER, + request_handler = ?NO_REQ_HANDLER, connect_timeout = ?DEFAULT_CONNECT_TIMEOUT, last_packet_id = 1}), {ok, initialized, init_parse_state(State)}. @@ -516,6 +530,8 @@ init([{name, Name} | Opts], State) -> init([{owner, Owner} | Opts], State) when is_pid(Owner) -> link(Owner), init(Opts, State#state{owner = Owner}); +init([{msg_handler, Hdlr} | Opts], State) -> + init(Opts, State#state{msg_handler = Hdlr}); init([{host, Host} | Opts], State) -> init(Opts, State#state{host = Host}); init([{port, Port} | Opts], State) -> @@ -857,12 +873,12 @@ connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> publish_process(?QOS_2, Packet, State); connected(cast, ?PUBACK_PACKET(PacketId, ReasonCode, Properties), - State = #state{owner = Owner, inflight = Inflight}) -> + State = #state{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} -> - Owner ! {puback, #{packet_id => PacketId, - reason_code => ReasonCode, - properties => Properties}}, + ok = eval_msg_handler(State, puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}), {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; none -> emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]), @@ -899,12 +915,12 @@ connected(cast, ?PUBREL_PACKET(PacketId), end; connected(cast, ?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), - State = #state{owner = Owner, inflight = Inflight}) -> + State = #state{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {pubrel, _PacketId, _Ts}} -> - Owner ! {puback, #{packet_id => PacketId, - reason_code => ReasonCode, - properties => Properties}}, + ok = eval_msg_handler(State, puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}), {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; none -> emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]), @@ -943,9 +959,8 @@ connected(cast, ?PACKET(?PINGRESP), State) -> false -> {keep_state, State} end; -connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), - State = #state{owner = Owner}) -> - Owner ! {disconnected, ReasonCode, Properties}, +connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), State) -> + ok = eval_msg_handler(State, disconnected, {ReasonCode, Properties}), {stop, disconnected, State}; connected(info, {timeout, _TRef, keepalive}, State = #state{force_ping = true}) -> @@ -1101,8 +1116,8 @@ assign_id(?NO_CLIENT_ID, Props) -> assign_id(Id, _Props) -> Id. -publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State = #state{auto_ack = AutoAck}) -> - _ = deliver(packet_to_msg(Packet), State), +publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State0 = #state{auto_ack = AutoAck}) -> + State = deliver(packet_to_msg(Packet), State0), case AutoAck of true -> send_puback(?PUBACK_PACKET(PacketId), State); false -> {keep_state, State} @@ -1116,18 +1131,11 @@ publish_process(?QOS_2, Packet = ?PUBLISH_PACKET(?QOS_2, PacketId), Stop -> Stop end. -response_publish(undefined, State, _QoS, _Payload) -> - State; -response_publish(Properties, State = #state{request_handler = RequestHandler}, QoS, Payload) -> - case maps:find('Response-Topic', Properties) of - {ok, ResponseTopic} -> - case RequestHandler of - ?NO_HANDLER -> State; - _ -> do_publish(ResponseTopic, Properties, State, QoS, Payload) - end; - _ -> - State - end. +response_publish(#{'Response-Topic' := ResponseTopic} = Properties, + State = #state{request_handler = RequestHandler}, QoS, Payload) + when RequestHandler =/= ?NO_REQ_HANDLER -> + do_publish(ResponseTopic, Properties, State, QoS, Payload); +response_publish(_Properties, State, _QoS, _Payload) -> State. do_publish(ResponseTopic, Properties, State = #state{request_handler = RequestHandler}, ?QOS_0, Payload) -> Msg = #mqtt_msg{qos = ?QOS_0, @@ -1251,19 +1259,33 @@ retry_send(pubrel, PacketId, Now, State = #state{inflight = Inflight}) -> Error end. +deliver(_Msg, State = #state{request_handler = Hdlr}) when Hdlr =/= ?NO_REQ_HANDLER -> + %% message has been terminated by request handler, hence should not continue processing + State; deliver(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId, topic = Topic, props = Props, payload = Payload}, - State = #state{owner = Owner, request_handler = RequestHandler}) -> - case RequestHandler of - ?NO_HANDLER -> - Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId, - topic => Topic, properties => Props, payload => Payload, - client_pid => self()}}; - _ -> - ok - end, + State) -> + Msg = #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId, + topic => Topic, properties => Props, payload => Payload, + client_pid => self()}, + ok = eval_msg_handler(State, publish, Msg), State. +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, + owner = Owner}, + disconnected, {ReasonCode, Properties}) -> + %% Special handling for disconnected message when there is no handler callback + Owner ! {disconnected, ReasonCode, Properties}, + ok; +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, + owner = Owner}, Kind, Msg) -> + Owner ! {Kind, Msg}, + ok; +eval_msg_handler(#state{msg_handler = Handler}, Kind, Msg) -> + F = maps:get(Kind, Handler), + _ = F(Msg), + ok. + packet_to_msg(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, dup = Dup, qos = QoS, From 6d51d78dfce87cdce740d926cb41bc1ba4584691 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 10 Feb 2019 14:44:43 +0100 Subject: [PATCH 03/26] Add portal transport over emqx_client. --- etc/emqx.conf | 18 +++-- src/emqx_portal_connect.erl | 5 +- src/portal/emqx_portal.erl | 29 ++++--- src/portal/emqx_portal_mqtt.erl | 135 ++++++++++++++++++++++++++++++++ src/portal/emqx_portal_msg.erl | 12 +-- src/portal/emqx_portal_rpc.erl | 6 +- test/emqx_portal_mqtt_tests.erl | 62 +++++++++++++++ test/emqx_portal_tests.erl | 15 ++-- 8 files changed, 249 insertions(+), 33 deletions(-) create mode 100644 src/portal/emqx_portal_mqtt.erl create mode 100644 test/emqx_portal_mqtt_tests.erl diff --git a/etc/emqx.conf b/etc/emqx.conf index 5a7a698d7..554238f2c 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1694,11 +1694,14 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: Number ## bridge.aws.subscription.2.qos = 1 -## Maximum number of messages in one batch for buffer queue to store +## Maximum number of messages in one batch when sending to remote borkers +## NOTE: when bridging viar MQTT connection to remote broker, this config is only +## used for internal message passing optimization as the underlying MQTT +## protocol does not supports batching. ## ## Value: Integer -## default: 1000 -## bridge.aws.queue.batch_size = 1000 +## default: 32 +## bridge.aws.queue.batch_size = 32 ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, @@ -1844,11 +1847,14 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Value: Number ## bridge.azure.subscription.2.qos = 1 -## Batch size for buffer queue stored +## Maximum number of messages in one batch when sending to remote borkers +## NOTE: when bridging viar MQTT connection to remote broker, this config is only +## used for internal message passing optimization as the underlying MQTT +## protocol does not supports batching. ## ## Value: Integer -## default: 1000 -## bridge.azure.queue.batch_size = 1000 +## default: 32 +## bridge.azure.queue.batch_size = 32 ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, diff --git a/src/emqx_portal_connect.erl b/src/emqx_portal_connect.erl index 35f4b53dc..ab3a3f5b8 100644 --- a/src/emqx_portal_connect.erl +++ b/src/emqx_portal_connect.erl @@ -20,11 +20,12 @@ -optional_callbacks([]). +%% map fields depend on implementation -type config() :: map(). -type connection() :: term(). -type conn_ref() :: term(). -type batch() :: emqx_protal:batch(). --type batch_ref() :: reference(). +-type ack_ref() :: emqx_portal:ack_ref(). -include("logger.hrl"). @@ -36,7 +37,7 @@ %% send to remote node/cluster %% portal worker (the caller process) should be expecting %% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster --callback send(connection(), batch()) -> {ok, batch_ref()} | {error, any()}. +-callback send(connection(), batch()) -> {ok, ack_ref()} | {error, any()}. %% called when owner is shutting down. -callback stop(conn_ref(), connection()) -> ok. diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 7e7d678f6..623f7f233 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -52,6 +52,9 @@ %% to support automatic load-balancing, i.e. in case it can not keep up %% with the amount of messages comming in, administrator should split and %% balance topics between worker/connections manually. +%% +%% NOTES: +%% * Local messages are all normalised to QoS-1 when exporting to remote -module(emqx_portal). -behaviour(gen_statem). @@ -71,17 +74,18 @@ -export_type([config/0, batch/0, - ref/0 + ack_ref/0 ]). -type config() :: map(). -type batch() :: [emqx_portal_msg:msg()]. --type ref() :: reference(). +-type ack_ref() :: term(). -include("logger.hrl"). -include("emqx_mqtt.hrl"). --define(DEFAULT_BATCH_COUNT, 100). +%% same as default in-flight limit for emqx_client +-define(DEFAULT_BATCH_COUNT, 32). -define(DEFAULT_BATCH_BYTES, 1 bsl 20). -define(DEFAULT_SEND_AHEAD, 8). -define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)). @@ -110,7 +114,7 @@ start_link(Name, Config) when is_list(Config) -> start_link(Name, maps:from_list(Config)); start_link(Name, Config) -> - gen_statem:start_link({local, Name}, ?MODULE, Config, []). + gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []). stop(Pid) -> gen_statem:stop(Pid). @@ -122,7 +126,7 @@ import_batch(Batch, AckFun) -> %% @doc This function is to be evaluated on message/batch exporter side %% when message/batch is accepted by remote node. --spec handle_ack(pid(), ref()) -> ok. +-spec handle_ack(pid(), ack_ref()) -> ok. handle_ack(Pid, Ref) when node() =:= node(Pid) -> Pid ! {batch_ack, Ref}, ok. @@ -231,7 +235,8 @@ connected(internal, maybe_send, State) -> end; connected(info, {disconnected, ConnRef, Reason}, #{conn_ref := ConnRef, connection := Conn} = State) -> - ?INFO("Portal ~p diconnected~nreason=~p", [Conn, Reason]), + ?INFO("Portal ~p diconnected~nreason=~p", + [name(), Conn, Reason]), {next_state, connecting, State#{conn_ref := undefined, connection := undefined @@ -255,7 +260,8 @@ common(_StateName, info, {dispatch, _, Msg}, NewQ = replayq:append(Q, collect([Msg])), {keep_state, State#{replayq => NewQ}, ?maybe_send}; common(StateName, Type, Content, State) -> - ?INFO("Ignored unknown ~p event ~p at state ~p", [Type, Content, StateName]), + ?DEBUG("Portal ~p discarded ~p type event at state ~p:~p", + [name(), Type, StateName, Content]), {keep_state, State}. collect(Acc) -> @@ -300,6 +306,7 @@ pop_and_send(#{replayq := Q, do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) -> case maybe_send(State, Batch) of {ok, Ref} -> + %% this is a list of inflight BATCHes, not expecting it to be too long NewInflight = Inflight ++ [#{q_ack_ref => QAckRef, send_ack_ref => Ref, batch => Batch @@ -326,8 +333,6 @@ subscribe_local_topics(Topics) -> emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}) end, Topics). -name() -> {_, Name} = process_info(self(), registered_name), Name. - disconnect(#{connection := Conn, conn_ref := ConnRef, connect_module := Module @@ -347,10 +352,14 @@ maybe_send(#{connect_module := Module, connection := Connection, mountpoint := Mountpoint }, Batch) -> - Module:send(Connection, [emqx_portal_msg:apply_mountpoint(M, Mountpoint) || M <- Batch]). + Module:send(Connection, [emqx_portal_msg:to_export(M, Mountpoint) || M <- Batch]). format_mountpoint(undefined) -> undefined; format_mountpoint(Prefix) -> binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). +name() -> {_, Name} = process_info(self(), registered_name), Name. + +name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). + diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl new file mode 100644 index 000000000..0ce5140b0 --- /dev/null +++ b/src/portal/emqx_portal_mqtt.erl @@ -0,0 +1,135 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc This module implements EMQX Portal transport layer on top of MQTT protocol + +-module(emqx_portal_mqtt). +-behaviour(emqx_portal_connect). + +%% behaviour callbacks +-export([start/1, + send/2, + stop/2 + ]). + +-include("emqx_mqtt.hrl"). + +-define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}). + +%% Messages towards ack collector process +-define(SENT(MaxPktId), {sent, MaxPktId}). +-define(ACKED(AnyPktId), {acked, AnyPktId}). +-define(STOP(Ref), {stop, Ref}). + +start(Config) -> + Ref = make_ref(), + Parent = self(), + AckCollector = spawn_link(fun() -> ack_collector(Parent, Ref) end), + Handlers = make_hdlr(Parent, AckCollector, Ref), + case emqx_client:start_link(Config#{msg_handler => Handlers, owner => AckCollector}) of + {ok, Pid} -> + case emqx_client:connect(Pid) of + {ok, _} -> + %% ack collector is always a new pid every reconnect. + %% use it as a connection reference + {ok, Ref, #{ack_collector => AckCollector, + client_pid => Pid}}; + {error, Reason} -> + ok = stop(AckCollector, Pid), + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +stop(Ref, #{ack_collector := AckCollector, + client_pid := Pid}) -> + MRef = monitor(process, AckCollector), + unlink(AckCollector), + _ = AckCollector ! ?STOP(Ref), + receive + {'DOWN', MRef, _, _, _} -> + ok + after + 1000 -> + exit(AckCollector, kill) + end, + _ = emqx_client:stop(Pid), + ok. + +send(#{client_pid := ClientPid, ack_collector := AckCollector}, Batch) -> + send_loop(ClientPid, AckCollector, Batch). + +send_loop(ClientPid, AckCollector, [Msg | Rest]) -> + case emqx_client:publish(ClientPid, Msg) of + {ok, PktId} when Rest =:= [] -> + Rest =:= [] andalso AckCollector ! ?SENT(PktId), + {ok, PktId}; + {ok, _PktId} -> + send_loop(ClientPid, AckCollector, Rest); + {error, {_PacketId, inflight_full}} -> + timer:sleep(100), + send_loop(ClientPid, AckCollector, [Msg | Rest]); + {error, Reason} -> + %% There is no partial sucess of a batch and recover from the middle + %% only to retry all messages in one batch + {error, Reason} + end. + +ack_collector(Parent, ConnRef) -> + ack_collector(Parent, ConnRef, []). + +ack_collector(Parent, ConnRef, PktIds) -> + NewIds = + receive + ?STOP(ConnRef) -> + exit(normal); + ?SENT(PktId) -> + %% this ++ only happens per-BATCH, hence no optimization + PktIds ++ [PktId]; + ?ACKED(PktId) -> + handle_ack(Parent, PktId, PktIds) + after + 200 -> + PktIds + end, + ack_collector(Parent, ConnRef, NewIds). + +handle_ack(Parent, PktId, [PktId | Rest]) -> + %% A batch is finished, time to ack portal + ok = emqx_portal:handle_ack(Parent, PktId), + Rest; +handle_ack(_Parent, PktId, [BatchMaxPktId | _] = All) -> + %% partial ack of a batch, terminate here. + true = (PktId < BatchMaxPktId), %% bad order otherwise + All. + +%% When puback for QoS-1 message is received from remote MQTT broker +%% NOTE: no support for QoS-2 +handle_puback(AckCollector, #{packet_id := PktId, reason_code := RC}) -> + RC =:= ?RC_SUCCESS andalso error(RC), + AckCollector ! ?ACKED(PktId), + ok. + +%% Message published from remote broker. Import to local broker. +import_msg(Msg) -> + %% auto-ack should be enabled in emqx_client, hence dummy ack-fun. + emqx_portal:import_batch([Msg], _AckFun = fun() -> ok end). + +make_hdlr(Parent, AckCollector, Ref) -> + #{puback => fun(Ack) -> handle_puback(AckCollector, Ack) end, + publish => fun(Msg) -> import_msg(Msg) end, + disconnected => fun(RC, _Properties) -> Parent ! {disconnected, Ref, RC}, ok end + }. + diff --git a/src/portal/emqx_portal_msg.erl b/src/portal/emqx_portal_msg.erl index af9cc1b01..252ea72d0 100644 --- a/src/portal/emqx_portal_msg.erl +++ b/src/portal/emqx_portal_msg.erl @@ -16,7 +16,7 @@ -export([ to_binary/1 , from_binary/1 - , apply_mountpoint/2 + , to_export/2 , to_broker_msgs/1 , estimate_size/1 ]). @@ -28,10 +28,12 @@ -type msg() :: emqx_types:message(). -%% @doc Mount topic to a prefix. --spec apply_mountpoint(msg(), undefined | binary()) -> msg(). -apply_mountpoint(#{topic := Topic} = Msg, Mountpoint) -> - Msg#{topic := topic(Mountpoint, Topic)}. +%% @doc Make export format: +%% 1. Mount topic to a prefix +%% 2. fix QoS to 1 +-spec to_export(msg(), undefined | binary()) -> msg(). +to_export(#{topic := Topic} = Msg, Mountpoint) -> + Msg#{topic := topic(Mountpoint, Topic), qos => 1}. %% @doc Make `binary()' in order to make iodata to be persisted on disk. -spec to_binary(msg()) -> binary(). diff --git a/src/portal/emqx_portal_rpc.erl b/src/portal/emqx_portal_rpc.erl index 8b1136353..fcd8b24e9 100644 --- a/src/portal/emqx_portal_rpc.erl +++ b/src/portal/emqx_portal_rpc.erl @@ -29,7 +29,7 @@ , heartbeat/2 ]). --type batch_ref() :: emqx_portal:batch_ref(). +-type ack_ref() :: emqx_portal:ack_ref(). -type batch() :: emqx_portal:batch(). -define(HEARTBEAT_INTERVAL, timer:seconds(1)). @@ -59,7 +59,7 @@ stop(Pid, _Remote) when is_pid(Pid) -> ok. %% @doc Callback for `emqx_portal_connect' behaviour --spec send(node(), batch()) -> {ok, batch_ref()} | {error, any()}. +-spec send(node(), batch()) -> {ok, ack_ref()} | {error, any()}. send(Remote, Batch) -> Sender = self(), case ?RPC:call(Remote, ?MODULE, handle_send, [Sender, Batch]) of @@ -68,7 +68,7 @@ send(Remote, Batch) -> end. %% @doc Handle send on receiver side. --spec handle_send(pid(), batch()) -> {ok, batch_ref()} | {error, any()}. +-spec handle_send(pid(), batch()) -> {ok, ack_ref()} | {error, any()}. handle_send(SenderPid, Batch) -> SenderNode = node(SenderPid), Ref = make_ref(), diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_portal_mqtt_tests.erl new file mode 100644 index 000000000..0312bca49 --- /dev/null +++ b/test/emqx_portal_mqtt_tests.erl @@ -0,0 +1,62 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_mqtt_tests). +-include_lib("eunit/include/eunit.hrl"). + +send_and_ack_test() -> + %% delegate from gen_rpc to rpc for unit test + Tester = self(), + meck:new(emqx_client, [passthrough, no_history]), + meck:expect(emqx_client, start_link, 1, + fun(#{msg_handler := Hdlr}) -> {ok, Hdlr} end), + meck:expect(emqx_client, connect, 1, {ok, dummy}), + meck:expect(emqx_client, stop, 1, ok), + meck:expect(emqx_client, publish, 2, + fun(_Conn, Msg) -> + case rand:uniform(100) of + 1 -> + {error, {dummy, inflight_full}}; + _ -> + Tester ! {published, Msg}, + {ok, Msg} + end + end), + try + Max = 100, + Batch = lists:seq(1, Max), + {ok, Ref, Conn} = emqx_portal_mqtt:start(#{}), + %% return last packet id as batch reference + {ok, AckRef} = emqx_portal_mqtt:send(Conn, Batch), + %% expect batch ack + {ok, LastId} = collect_acks(Conn, Batch), + %% asset received ack matches the batch ref returned in send API + ?assertEqual(AckRef, LastId), + ok = emqx_portal_mqtt:stop(Ref, Conn) + after + meck:unload(emqx_client) + end. + +collect_acks(_Conn, []) -> + receive {batch_ack, Id} -> {ok, Id} end; +collect_acks(#{client_pid := Client} = Conn, [Id | Rest]) -> + %% mocked for testing, should be a pid() at runtime + #{puback := PubAckCallback} = Client, + receive + {published, Id} -> + PubAckCallback(#{packet_id => Id, reason_code => dummy}), + collect_acks(Conn, Rest) + end. + + diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index 5826a327e..85975da3e 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -18,7 +18,8 @@ -include_lib("eunit/include/eunit.hrl"). -include("emqx_mqtt.hrl"). --define(PORTAL_NAME, test_portal). +-define(PORTAL_NAME, test). +-define(PORTAL_REG_NAME, emqx_portal_test). -define(WAIT(PATTERN, TIMEOUT), receive PATTERN -> @@ -49,11 +50,11 @@ reconnect_test() -> Config = make_config(Ref, self(), {error, test}), {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), %% assert name registered - ?assertEqual(Pid, whereis(?PORTAL_NAME)), + ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), ?WAIT({connection_start_attempt, Ref}, 1000), %% expect same message again ?WAIT({connection_start_attempt, Ref}, 1000), - ok = emqx_portal:stop(?PORTAL_NAME), + ok = emqx_portal:stop(?PORTAL_REG_NAME), ok. %% connect first, disconnect, then connect again @@ -61,11 +62,11 @@ disturbance_test() -> Ref = make_ref(), Config = make_config(Ref, self(), {ok, Ref, connection}), {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), - ?assertEqual(Pid, whereis(?PORTAL_NAME)), + ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), ?WAIT({connection_start_attempt, Ref}, 1000), Pid ! {disconnected, Ref, test}, ?WAIT({connection_start_attempt, Ref}, 1000), - ok = emqx_portal:stop(?PORTAL_NAME). + ok = emqx_portal:stop(?PORTAL_REG_NAME). %% buffer should continue taking in messages when disconnected buffer_when_disconnected_test_() -> @@ -88,11 +89,11 @@ test_buffer_when_disconnected() -> {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), Sender ! {portal, Pid}, Receiver ! {portal, Pid}, - ?assertEqual(Pid, whereis(?PORTAL_NAME)), + ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), Pid ! {disconnected, Ref, test}, ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 2000), ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), - ok = emqx_portal:stop(?PORTAL_NAME). + ok = emqx_portal:stop(?PORTAL_REG_NAME). %% Feed messages to portal sender_loop(_Pid, [], _) -> exit(normal); From b9e8bde3b0d9b4be56a0c011d47152dc82b1bad7 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 10 Feb 2019 23:57:51 +0100 Subject: [PATCH 04/26] Add first CT test for emqx_portal based on rpc --- src/portal/emqx_portal_msg.erl | 10 +++-- test/emqx_ct_helpers.erl | 49 ++++++++++++++++++++++- test/emqx_portal_SUITE.erl | 73 ++++++++++++++++++++++++++++++++++ test/emqx_shared_sub_SUITE.erl | 48 +--------------------- 4 files changed, 129 insertions(+), 51 deletions(-) create mode 100644 test/emqx_portal_SUITE.erl diff --git a/src/portal/emqx_portal_msg.erl b/src/portal/emqx_portal_msg.erl index 252ea72d0..12f5926a3 100644 --- a/src/portal/emqx_portal_msg.erl +++ b/src/portal/emqx_portal_msg.erl @@ -32,8 +32,8 @@ %% 1. Mount topic to a prefix %% 2. fix QoS to 1 -spec to_export(msg(), undefined | binary()) -> msg(). -to_export(#{topic := Topic} = Msg, Mountpoint) -> - Msg#{topic := topic(Mountpoint, Topic), qos => 1}. +to_export(#message{topic = Topic} = Msg, Mountpoint) -> + Msg#message{topic = topic(Mountpoint, Topic), qos = 1}. %% @doc Make `binary()' in order to make iodata to be persisted on disk. -spec to_binary(msg()) -> binary(). @@ -46,15 +46,19 @@ from_binary(Bin) -> binary_to_term(Bin). %% @doc Estimate the size of a message. %% Count only the topic length + payload size -spec estimate_size(msg()) -> integer(). -estimate_size(#{topic := Topic, payload := Payload}) -> +estimate_size(#message{topic = Topic, payload = Payload}) -> size(Topic) + size(Payload). %% @doc By message/batch receiver, transform received batch into %% messages to dispatch to local brokers. to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch). +to_broker_msg(#message{} = Msg) -> + %% internal format from another EMQX node via rpc + Msg; to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, properties := Props, payload := Payload}) -> + %% published from remote node over a MQTT connection emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, emqx_message:make(portal, QoS, Topic, Payload))). diff --git a/test/emqx_ct_helpers.erl b/test/emqx_ct_helpers.erl index eae22d6ab..361a6b4a9 100644 --- a/test/emqx_ct_helpers.erl +++ b/test/emqx_ct_helpers.erl @@ -14,9 +14,56 @@ -module(emqx_ct_helpers). --export([ensure_mnesia_stopped/0]). +-export([ensure_mnesia_stopped/0, wait_for/4]). ensure_mnesia_stopped() -> ekka_mnesia:ensure_stopped(), ekka_mnesia:delete_schema(). +%% Help function to wait for Fun to yeild 'true'. +wait_for(Fn, Ln, F, Timeout) -> + {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), + wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). + +wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> + receive + {'DOWN', Mref, process, Pid, normal} -> + ok; + {'DOWN', Mref, process, Pid, {unexpected, Result}} -> + erlang:error({unexpected, Fn, Ln, Result}); + {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> + erlang:raise(C, {Fn, Ln, E}, S) + after + Timeout -> + case Kill of + true -> + erlang:demonitor(Mref, [flush]), + erlang:exit(Pid, kill), + erlang:error({Fn, Ln, timeout}); + false -> + Pid ! stop, + wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) + end + end. + +wait_loop(_F, ok) -> exit(normal); +wait_loop(F, LastRes) -> + receive + stop -> erlang:exit(LastRes) + after + 100 -> + Res = catch_call(F), + wait_loop(F, Res) + end. + +catch_call(F) -> + try + case F() of + true -> ok; + Other -> {unexpected, Other} + end + catch + C : E : S -> + {crashed, {C, E, S}} + end. + diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl new file mode 100644 index 000000000..21effdb08 --- /dev/null +++ b/test/emqx_portal_SUITE.erl @@ -0,0 +1,73 @@ +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_portal_SUITE). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([t_rpc/1, + t_mqtt/1 + ]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include("emqx_mqtt.hrl"). +-include("emqx.hrl"). + +-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). + +all() -> [t_rpc, + t_mqtt + ]. + +init_per_suite(Config) -> + case node() of + nonode@nohost -> + net_kernel:start(['emqx@127.0.0.1', longnames]); + _ -> + ok + end, + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +t_rpc(Config) when is_list(Config) -> + Cfg = #{address => node(), + forwards => [<<"t_rpc/#">>], + connect_module => emqx_portal_rpc, + mountpoint => <<"forwarded">> + }, + {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), + ClientId = <<"ClientId">>, + try + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + %% message from a different client, to avoid getting terminated by no-local + Msg1 = emqx_message:make(<<"ClientId-2">>, ?QOS_2, <<"t_rpc/one">>, <<"hello">>), + ok = emqx_session:subscribe(SPid, [{<<"forwarded/t_rpc/one">>, #{qos => ?QOS_1}}]), + PacketId = 1, + emqx_session:publish(SPid, PacketId, Msg1), + ?wait(case emqx_mock_client:get_last_message(ConnPid) of + {publish, PacketId, #message{topic = <<"forwarded/t_rpc/one">>}} -> true; + Other -> Other + end, 4000), + emqx_mock_client:close_session(ConnPid) + after + ok = emqx_portal:stop(Pid) + end. + +t_mqtt(Config) when is_list(Config) -> ok. + + diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 1ee059812..41537e58a 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -29,7 +29,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(wait(For, Timeout), wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). +-define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). all() -> [t_random_basic, t_random, @@ -259,49 +259,3 @@ ensure_config(Strategy, AckEnabled) -> subscribed(Group, Topic, Pid) -> lists:member(Pid, emqx_shared_sub:subscribers(Group, Topic)). -wait_for(Fn, Ln, F, Timeout) -> - {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), - wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). - -wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> - receive - {'DOWN', Mref, process, Pid, normal} -> - ok; - {'DOWN', Mref, process, Pid, {unexpected, Result}} -> - erlang:error({unexpected, Fn, Ln, Result}); - {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> - erlang:raise(C, {Fn, Ln, E}, S) - after - Timeout -> - case Kill of - true -> - erlang:demonitor(Mref, [flush]), - erlang:exit(Pid, kill), - erlang:error({Fn, Ln, timeout}); - false -> - Pid ! stop, - wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) - end - end. - -wait_loop(_F, ok) -> exit(normal); -wait_loop(F, LastRes) -> - receive - stop -> erlang:exit(LastRes) - after - 100 -> - Res = catch_call(F), - wait_loop(F, Res) - end. - -catch_call(F) -> - try - case F() of - true -> ok; - Other -> {unexpected, Other} - end - catch - C : E : S -> - {crashed, {C, E, S}} - end. - From 67376727c96ffe425492e9f2c0e102ae19f6299e Mon Sep 17 00:00:00 2001 From: spring2maz Date: Fri, 15 Feb 2019 12:36:31 +0100 Subject: [PATCH 05/26] Add batch send support for emqx_client:publish/2 also cover emqx_portal_mqtt with CT --- include/emqx_mqtt.hrl | 8 ++- src/emqx_client.erl | 54 +++++++++++----- src/emqx_topic.erl | 3 +- src/portal/emqx_portal.erl | 4 +- src/portal/emqx_portal_mqtt.erl | 110 +++++++++++++++++++------------- src/portal/emqx_portal_msg.erl | 26 ++++++-- test/emqx_portal_SUITE.erl | 71 ++++++++++++++++++++- test/emqx_portal_mqtt_tests.erl | 49 +++++++++----- test/emqx_portal_tests.erl | 11 +--- 9 files changed, 243 insertions(+), 93 deletions(-) diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 7e7670112..3bba42216 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -171,10 +171,16 @@ -define(RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED, 16#A2). %%-------------------------------------------------------------------- -%% Maximum MQTT Packet Length +%% Maximum MQTT Packet ID and Length %%-------------------------------------------------------------------- +-define(MAX_PACKET_ID, 16#ffff). -define(MAX_PACKET_SIZE, 16#fffffff). +-define(BUMP_PACKET_ID(Base, Bump), + case Base + Bump of + __I__ when __I__ > ?MAX_PACKET_ID -> __I__ - ?MAX_PACKET_ID; + __I__ -> __I__ + end). %%-------------------------------------------------------------------- %% MQTT Frame Mask diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 389c9e902..71527871f 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -389,8 +389,8 @@ publish(Client, Topic, Properties, Payload, Opts) props = Properties, payload = iolist_to_binary(Payload)}). --spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}). -publish(Client, Msg) when is_record(Msg, mqtt_msg) -> +-spec(publish(client(), #mqtt_msg{} | [#mqtt_msg{}]) -> ok | {ok, packet_id()} | {error, term()}). +publish(Client, Msg) -> gen_statem:call(Client, {publish, Msg}). -spec(unsubscribe(client(), topic() | [topic()]) -> subscribe_ret()). @@ -756,9 +756,6 @@ connected({call, From}, pause, State) -> connected({call, From}, resume, State) -> {keep_state, State#state{paused = false}, [{reply, From, ok}]}; -connected({call, From}, stop, _State) -> - {stop_and_reply, normal, [{reply, From, ok}]}; - connected({call, From}, get_properties, State = #state{properties = Properties}) -> {keep_state, State, [{reply, From, Properties}]}; @@ -790,19 +787,22 @@ connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) -> {stop_and_reply, Reason, [{reply, From, Error}]} end; -connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, - State = #state{inflight = Inflight, last_packet_id = PacketId}) +connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, State) when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> + connected({call, From}, {publish, [Msg]}, State); + +%% when publishing a batch, {ok, BasePacketId} is returned, +%% following packet ids for the batch tail are mod (1 bsl 16) consecutive +connected({call, From}, {publish, Msgs}, + State = #state{inflight = Inflight, last_packet_id = PacketId}) when is_list(Msgs) -> + %% NOTE: to ensure API call atomicity, inflight buffer may overflow case emqx_inflight:is_full(Inflight) of true -> - {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]}; + {keep_state, State, [{reply, From, {error, inflight_full}}]}; false -> - Msg1 = Msg#mqtt_msg{packet_id = PacketId}, - case send(Msg1, State) of + case send_batch(assign_packet_id(Msgs, PacketId), State) of {ok, NewState} -> - Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), - {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), - [{reply, From, {ok, PacketId}}]}; + {keep_state, ensure_retry_timer(NewState), [{reply, From, {ok, PacketId}}]}; {error, Reason} -> {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} end @@ -1011,6 +1011,8 @@ should_ping(Sock) -> Error end. +handle_event({call, From}, stop, _StateName, _State) -> + {stop_and_reply, normal, [{reply, From, ok}]}; handle_event(info, {TcpOrSsL, _Sock, Data}, _StateName, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> emqx_logger:debug("RECV Data: ~p", [Data]), @@ -1333,6 +1335,17 @@ send_puback(Packet, State) -> {error, Reason} -> {stop, {shutdown, Reason}} end. +send_batch([], State) -> {ok, State}; +send_batch([Msg = #mqtt_msg{packet_id = PacketId} | Rest], + State = #state{inflight = Inflight}) -> + case send(Msg, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg, os:timestamp()}, Inflight), + send_batch(Rest, NewState#state{inflight = Inflight1}); + {error, Reason} -> + {error, Reason} + end. + send(Msg, State) when is_record(Msg, mqtt_msg) -> send(msg_to_packet(Msg), State); @@ -1375,10 +1388,17 @@ next_events(Packets) -> [{next_event, cast, Packet} || Packet <- lists:reverse(Packets)]. %%------------------------------------------------------------------------------ -%% Next packet id +%% packet_id generation and assignment -next_packet_id(State = #state{last_packet_id = 16#ffff}) -> - State#state{last_packet_id = 1}; +assign_packet_id(Msg = #mqtt_msg{}, Id) -> + Msg#mqtt_msg{packet_id = Id}; +assign_packet_id([H | T], Id) -> + [assign_packet_id(H, Id) | assign_packet_id(T, next_packet_id(Id))]; +assign_packet_id([], _Id) -> + []. next_packet_id(State = #state{last_packet_id = Id}) -> - State#state{last_packet_id = Id + 1}. + State#state{last_packet_id = next_packet_id(Id)}; +next_packet_id(16#ffff) -> 1; +next_packet_id(Id) -> Id + 1. + diff --git a/src/emqx_topic.erl b/src/emqx_topic.erl index 4c90c3f39..bb615ccfe 100644 --- a/src/emqx_topic.erl +++ b/src/emqx_topic.erl @@ -144,7 +144,8 @@ prepend(Parent0, W) -> bin('') -> <<>>; bin('+') -> <<"+">>; bin('#') -> <<"#">>; -bin(B) when is_binary(B) -> B. +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L). levels(Topic) when is_binary(Topic) -> length(words(Topic)). diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 623f7f233..51ce4a4dc 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -78,7 +78,7 @@ ]). -type config() :: map(). --type batch() :: [emqx_portal_msg:msg()]. +-type batch() :: [emqx_portal_msg:exp_msg()]. -type ack_ref() :: term(). -include("logger.hrl"). @@ -352,7 +352,7 @@ maybe_send(#{connect_module := Module, connection := Connection, mountpoint := Mountpoint }, Batch) -> - Module:send(Connection, [emqx_portal_msg:to_export(M, Mountpoint) || M <- Batch]). + Module:send(Connection, [emqx_portal_msg:to_export(Module, Mountpoint, M) || M <- Batch]). format_mountpoint(undefined) -> undefined; diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index 0ce5140b0..f01633111 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -28,7 +28,8 @@ -define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}). %% Messages towards ack collector process --define(SENT(MaxPktId), {sent, MaxPktId}). +-define(RANGE(Min, Max), {Min, Max}). +-define(SENT(PktIdRange), {sent, PktIdRange}). -define(ACKED(AnyPktId), {acked, AnyPktId}). -define(STOP(Ref), {stop, Ref}). @@ -41,10 +42,17 @@ start(Config) -> {ok, Pid} -> case emqx_client:connect(Pid) of {ok, _} -> - %% ack collector is always a new pid every reconnect. - %% use it as a connection reference - {ok, Ref, #{ack_collector => AckCollector, - client_pid => Pid}}; + try + subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])), + %% ack collector is always a new pid every reconnect. + %% use it as a connection reference + {ok, Ref, #{ack_collector => AckCollector, + client_pid => Pid}} + catch + throw : Reason -> + ok = stop(AckCollector, Pid), + {error, Reason} + end; {error, Reason} -> ok = stop(AckCollector, Pid), {error, Reason} @@ -53,72 +61,79 @@ start(Config) -> {error, Reason} end. -stop(Ref, #{ack_collector := AckCollector, - client_pid := Pid}) -> - MRef = monitor(process, AckCollector), - unlink(AckCollector), - _ = AckCollector ! ?STOP(Ref), +stop(Ref, #{ack_collector := AckCollector, client_pid := Pid}) -> + safe_stop(AckCollector, fun() -> AckCollector ! ?STOP(Ref) end, 1000), + safe_stop(Pid, fun() -> emqx_client:stop(Pid) end, 1000), + ok. + +safe_stop(Pid, StopF, Timeout) -> + MRef = monitor(process, Pid), + unlink(Pid), + try + StopF() + catch + _ : _ -> + ok + end, receive {'DOWN', MRef, _, _, _} -> ok after - 1000 -> - exit(AckCollector, kill) - end, - _ = emqx_client:stop(Pid), - ok. + Timeout -> + exit(Pid, kill) + end. -send(#{client_pid := ClientPid, ack_collector := AckCollector}, Batch) -> - send_loop(ClientPid, AckCollector, Batch). - -send_loop(ClientPid, AckCollector, [Msg | Rest]) -> - case emqx_client:publish(ClientPid, Msg) of - {ok, PktId} when Rest =:= [] -> - Rest =:= [] andalso AckCollector ! ?SENT(PktId), - {ok, PktId}; - {ok, _PktId} -> - send_loop(ClientPid, AckCollector, Rest); +send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, Batch) -> + case emqx_client:publish(ClientPid, Batch) of + {ok, BasePktId} -> + LastPktId = ?BUMP_PACKET_ID(BasePktId, length(Batch) - 1), + AckCollector ! ?SENT(?RANGE(BasePktId, LastPktId)), + %% return last pakcet id as batch reference + {ok, LastPktId}; {error, {_PacketId, inflight_full}} -> timer:sleep(100), - send_loop(ClientPid, AckCollector, [Msg | Rest]); + send(Conn, Batch); {error, Reason} -> - %% There is no partial sucess of a batch and recover from the middle + %% NOTE: There is no partial sucess of a batch and recover from the middle %% only to retry all messages in one batch {error, Reason} end. ack_collector(Parent, ConnRef) -> - ack_collector(Parent, ConnRef, []). + ack_collector(Parent, ConnRef, queue:new(), []). -ack_collector(Parent, ConnRef, PktIds) -> - NewIds = +ack_collector(Parent, ConnRef, Acked, Sent) -> + {NewAcked, NewSent} = receive ?STOP(ConnRef) -> exit(normal); - ?SENT(PktId) -> - %% this ++ only happens per-BATCH, hence no optimization - PktIds ++ [PktId]; ?ACKED(PktId) -> - handle_ack(Parent, PktId, PktIds) + match_acks(Parent, queue:in(PktId, Acked), Sent); + ?SENT(Range) -> + %% this message only happens per-batch, hence ++ is ok + match_acks(Parent, Acked, Sent ++ [Range]) after 200 -> - PktIds + {Acked, Sent} end, - ack_collector(Parent, ConnRef, NewIds). + ack_collector(Parent, ConnRef, NewAcked, NewSent). -handle_ack(Parent, PktId, [PktId | Rest]) -> - %% A batch is finished, time to ack portal +match_acks(_Parent, Acked, []) -> {Acked, []}; +match_acks(Parent, Acked, Sent) -> + match_acks_1(Parent, queue:out(Acked), Sent). + +match_acks_1(_Parent, {empty, Empty}, Sent) -> {Empty, Sent}; +match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, PktId) | Sent]) -> + %% batch finished ok = emqx_portal:handle_ack(Parent, PktId), - Rest; -handle_ack(_Parent, PktId, [BatchMaxPktId | _] = All) -> - %% partial ack of a batch, terminate here. - true = (PktId < BatchMaxPktId), %% bad order otherwise - All. + match_acks(Parent, Acked, Sent); +match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, Max) | Sent]) -> + match_acks(Parent, Acked, [?RANGE(PktId + 1, Max) | Sent]). %% When puback for QoS-1 message is received from remote MQTT broker %% NOTE: no support for QoS-2 handle_puback(AckCollector, #{packet_id := PktId, reason_code := RC}) -> - RC =:= ?RC_SUCCESS andalso error(RC), + RC =:= ?RC_SUCCESS orelse error({puback_error_code, RC}), AckCollector ! ?ACKED(PktId), ok. @@ -133,3 +148,10 @@ make_hdlr(Parent, AckCollector, Ref) -> disconnected => fun(RC, _Properties) -> Parent ! {disconnected, Ref, RC}, ok end }. +subscribe_remote_topics(ClientPid, Subscriptions) -> + [case emqx_client:subscribe(ClientPid, {bin(Topic), Qos}) of + {ok, _, _} -> ok; + Error -> throw(Error) + end || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})]. + +bin(L) -> iolist_to_binary(L). diff --git a/src/portal/emqx_portal_msg.erl b/src/portal/emqx_portal_msg.erl index 12f5926a3..f8554f0b6 100644 --- a/src/portal/emqx_portal_msg.erl +++ b/src/portal/emqx_portal_msg.erl @@ -16,7 +16,7 @@ -export([ to_binary/1 , from_binary/1 - , to_export/2 + , to_export/3 , to_broker_msgs/1 , estimate_size/1 ]). @@ -25,14 +25,32 @@ -include("emqx.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_client.hrl"). -type msg() :: emqx_types:message(). +-type exp_msg() :: emqx_types:message() | #mqtt_msg{}. %% @doc Make export format: %% 1. Mount topic to a prefix -%% 2. fix QoS to 1 --spec to_export(msg(), undefined | binary()) -> msg(). -to_export(#message{topic = Topic} = Msg, Mountpoint) -> +%% 2. Fix QoS to 1 +%% @end +%% Shame that we have to know the callback module here +%% would be great if we can get rid of #mqtt_msg{} record +%% and use #message{} in all places. +-spec to_export(emqx_portal_rpc | emqx_portal_mqtt, + undefined | binary(), msg()) -> exp_msg(). +to_export(emqx_portal_mqtt, Mountpoint, + #message{topic = Topic, + payload = Payload, + flags = Flags + }) -> + Retain = maps:get(retain, Flags, false), + #mqtt_msg{qos = ?QOS_1, + retain = Retain, + topic = topic(Mountpoint, Topic), + payload = Payload}; +to_export(_Module, Mountpoint, + #message{topic = Topic} = Msg) -> Msg#message{topic = topic(Mountpoint, Topic), qos = 1}. %% @doc Make `binary()' in order to make iodata to be persisted on disk. diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 21effdb08..f8ca04ea2 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -43,6 +43,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). +%% A loopback RPC to local node t_rpc(Config) when is_list(Config) -> Cfg = #{address => node(), forwards => [<<"t_rpc/#">>], @@ -68,6 +69,74 @@ t_rpc(Config) when is_list(Config) -> ok = emqx_portal:stop(Pid) end. -t_mqtt(Config) when is_list(Config) -> ok. +t_mqtt(Config) when is_list(Config) -> + SendToTopic = <<"t_mqtt/one">>, + Mountpoint = <<"forwarded/${node}/">>, + ForwardedTopic = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic]), + Cfg = #{address => "127.0.0.1:1883", + forwards => [SendToTopic], + connect_module => emqx_portal_mqtt, + mountpoint => Mountpoint, + username => "user", + clean_start => true, + client_id => "bridge_aws", + keepalive => 60000, + max_inflight => 32, + password => "passwd", + proto_ver => mqttv4, + queue => #{replayq_dir => "data/t_mqtt/", + replayq_seg_bytes => 10000, + batch_bytes_limit => 1000, + batch_count_limit => 10 + }, + reconnect_delay_ms => 1000, + ssl => false, + start_type => manual, + %% Consume back to forwarded message for verification + %% NOTE: this is a indefenite loopback without mocking emqx_portal:import_batch/2 + subscriptions => [{ForwardedTopic, 1}] + }, + Tester = self(), + Ref = make_ref(), + meck:new(emqx_portal, [passthrough, no_history]), + meck:expect(emqx_portal, import_batch, 2, + fun(Batch, AckFun) -> + Tester ! {Ref, Batch}, + AckFun() + end), + {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), + ClientId = <<"client-1">>, + try + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + %% message from a different client, to avoid getting terminated by no-local + Msgs = lists:seq(1, 10), + lists:foreach(fun(I) -> + Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic, integer_to_binary(I)), + emqx_session:publish(SPid, I, Msg) + end, Msgs), + ok = receive_and_match_messages(Ref, Msgs), + emqx_mock_client:close_session(ConnPid) + after + ok = emqx_portal:stop(Pid), + meck:unload(emqx_portal) + end. +receive_and_match_messages(Ref, Msgs) -> + TRef = erlang:send_after(timer:seconds(4), self(), {Ref, timeout}), + try + do_receive_and_match_messages(Ref, Msgs) + after + erlang:cancel_timer(TRef) + end, + ok. + +do_receive_and_match_messages(_Ref, []) -> ok; +do_receive_and_match_messages(Ref, [I | Rest]) -> + receive + {Ref, timeout} -> erlang:error(timeout); + {Ref, [#{payload := P}]} -> + ?assertEqual(I, binary_to_integer(P)), + do_receive_and_match_messages(Ref, Rest) + end. diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_portal_mqtt_tests.erl index 0312bca49..8f513a853 100644 --- a/test/emqx_portal_mqtt_tests.erl +++ b/test/emqx_portal_mqtt_tests.erl @@ -14,23 +14,28 @@ -module(emqx_portal_mqtt_tests). -include_lib("eunit/include/eunit.hrl"). +-include("emqx_mqtt.hrl"). send_and_ack_test() -> %% delegate from gen_rpc to rpc for unit test Tester = self(), meck:new(emqx_client, [passthrough, no_history]), meck:expect(emqx_client, start_link, 1, - fun(#{msg_handler := Hdlr}) -> {ok, Hdlr} end), + fun(#{msg_handler := Hdlr}) -> + {ok, spawn_link(fun() -> fake_client(Hdlr) end)} + end), meck:expect(emqx_client, connect, 1, {ok, dummy}), - meck:expect(emqx_client, stop, 1, ok), + meck:expect(emqx_client, stop, 1, + fun(Pid) -> Pid ! stop end), meck:expect(emqx_client, publish, 2, - fun(_Conn, Msg) -> + fun(_Conn, Msgs) -> case rand:uniform(100) of 1 -> {error, {dummy, inflight_full}}; _ -> - Tester ! {published, Msg}, - {ok, Msg} + BaseId = hd(Msgs), + Tester ! {published, Msgs}, + {ok, BaseId} end end), try @@ -39,24 +44,38 @@ send_and_ack_test() -> {ok, Ref, Conn} = emqx_portal_mqtt:start(#{}), %% return last packet id as batch reference {ok, AckRef} = emqx_portal_mqtt:send(Conn, Batch), + %% as if the remote broker replied with puback + ok = fake_pubacks(Conn), %% expect batch ack - {ok, LastId} = collect_acks(Conn, Batch), + AckRef1= receive {batch_ack, Id} -> Id end, %% asset received ack matches the batch ref returned in send API - ?assertEqual(AckRef, LastId), + ?assertEqual(AckRef, AckRef1), ok = emqx_portal_mqtt:stop(Ref, Conn) after meck:unload(emqx_client) end. -collect_acks(_Conn, []) -> - receive {batch_ack, Id} -> {ok, Id} end; -collect_acks(#{client_pid := Client} = Conn, [Id | Rest]) -> - %% mocked for testing, should be a pid() at runtime - #{puback := PubAckCallback} = Client, +fake_pubacks(#{client_pid := Client}) -> + #{puback := PubAckCallback} = get_hdlr(Client), receive - {published, Id} -> - PubAckCallback(#{packet_id => Id, reason_code => dummy}), - collect_acks(Conn, Rest) + {published, Msgs} -> + lists:foreach( + fun(Id) -> + PubAckCallback(#{packet_id => Id, reason_code => ?RC_SUCCESS}) + end, Msgs) + end. + +get_hdlr(Client) -> + Client ! {get_hdlr, self()}, + receive {hdr, Hdlr} -> Hdlr end. + +fake_client(Hdlr) -> + receive + {get_hdlr, Pid} -> + Pid ! {hdr, Hdlr}, + fake_client(Hdlr); + stop -> + exit(normal) end. diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index 85975da3e..3b2879b12 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -16,6 +16,7 @@ -behaviour(emqx_portal_connect). -include_lib("eunit/include/eunit.hrl"). +-include("emqx.hrl"). -include("emqx_mqtt.hrl"). -define(PORTAL_NAME, test). @@ -120,7 +121,7 @@ random_sleep(MaxInterval) -> end. match_nums([], Rest) -> Rest; -match_nums([#{payload := P} | Rest], Nums) -> +match_nums([#message{payload = P} | Rest], Nums) -> I = binary_to_integer(P), case Nums of [I | NumsLeft] -> match_nums(Rest, NumsLeft); @@ -137,11 +138,5 @@ make_config(Ref, TestPid, Result) -> make_msg(I) -> Payload = integer_to_binary(I), - #{qos => ?QOS_1, - dup => false, - retain => false, - topic => <<"test/topic">>, - properties => [], - payload => Payload - }. + emqx_message:make(<<"test/topic">>, Payload). From 9e78c186811808fdde67b0998323d3031762dad6 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Fri, 15 Feb 2019 17:06:02 +0100 Subject: [PATCH 06/26] Add get_forwards and get_subscriptions protal APIs --- src/emqx_types.erl | 2 +- src/portal/emqx_portal.erl | 30 +++++++++++++++++++++++++++--- src/portal/emqx_portal_mqtt.erl | 11 ++++++----- test/emqx_portal_SUITE.erl | 24 ++++++++++++++++++++++-- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/emqx_types.erl b/src/emqx_types.erl index d84b1099a..904e1df91 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -31,7 +31,7 @@ -type(pubsub() :: publish | subscribe). -type(topic() :: binary()). -type(subid() :: binary() | atom()). --type(subopts() :: #{qos := integer(), +-type(subopts() :: #{qos := eqmx_mqtt_types:qos(), share => binary(), atom() => term() }). diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 51ce4a4dc..238b6681f 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -41,7 +41,7 @@ %% | | | %% '--(1)---'--------(3)------' %% -%% (1): timeout +%% (1): retry timeout %% (2): successfuly connected to remote node/cluster %% (3): received {disconnected, conn_ref(), Reason} OR %% failed to send to remote node/cluster. @@ -72,11 +72,17 @@ %% state functions -export([connecting/3, connected/3]). +%% management APIs +-export([get_forwards/1]). %, add_forward/2, del_forward/2]). +-export([get_subscriptions/1]). %, add_subscription/3, del_subscription/2]). + -export_type([config/0, batch/0, ack_ref/0 ]). +-type id() :: atom() | string() | pid(). +-type qos() :: emqx_mqtt_types:qos(). -type config() :: map(). -type batch() :: [emqx_portal_msg:exp_msg()]. -type ack_ref() :: term(). @@ -131,6 +137,11 @@ handle_ack(Pid, Ref) when node() =:= node(Pid) -> Pid ! {batch_ack, Ref}, ok. +-spec get_forwards(id()) -> [emqx_topic:topic()]. +get_forwards(Id) -> gen_statem:call(id(Id), get_forwards). + +-spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. +get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). callback_mode() -> [state_functions, state_enter]. @@ -150,7 +161,12 @@ init(Config) -> end, Queue = replayq:open(QueueConfig#{sizer => fun emqx_portal_msg:estimate_size/1, marshaller => fun msg_marshaller/1}), - Topics = Get(forwards, []), + Topics = lists:sort([iolist_to_binary(T) || T <- Get(forwards, [])]), + Subs = lists:keysort(1, lists:map(fun({T0, QoS}) -> + T = iolist_to_binary(T0), + true = emqx_topic:validate({filter, T}), + {T, QoS} + end, Get(subscriptions, []))), ok = subscribe_local_topics(Topics), ConnectModule = maps:get(connect_module, Config), ConnectConfig = maps:without([connect_module, @@ -159,7 +175,7 @@ init(Config) -> max_inflight_batches, mountpoint, forwards - ], Config), + ], Config#{subscriptions => Subs}), ConnectFun = fun() -> emqx_portal_connect:start(ConnectModule, ConnectConfig) end, {ok, connecting, #{connect_module => ConnectModule, @@ -170,6 +186,7 @@ init(Config) -> max_inflight_batches => Get(max_inflight_batches, ?DEFAULT_SEND_AHEAD), mountpoint => format_mountpoint(Get(mountpoint, undefined)), topics => Topics, + subscriptions => Subs, replayq => Queue, inflight => [] }}. @@ -255,6 +272,10 @@ connected(Type, Content, State) -> common(connected, Type, Content, State). %% Common handlers +common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> + {keep_state_and_data, [{reply, From, Forwards}]}; +common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> + {keep_state_and_data, [{reply, From, Subs}]}; common(_StateName, info, {dispatch, _, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, collect([Msg])), @@ -363,3 +384,6 @@ name() -> {_, Name} = process_info(self(), registered_name), Name. name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). +id(Pid) when is_pid(Pid) -> Pid; +id(Name) -> name(Name). + diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index f01633111..ba7461943 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -149,9 +149,10 @@ make_hdlr(Parent, AckCollector, Ref) -> }. subscribe_remote_topics(ClientPid, Subscriptions) -> - [case emqx_client:subscribe(ClientPid, {bin(Topic), Qos}) of - {ok, _, _} -> ok; - Error -> throw(Error) - end || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})]. + lists:foreach(fun({Topic, Qos}) -> + case emqx_client:subscribe(ClientPid, Topic, Qos) of + {ok, _, _} -> ok; + Error -> throw(Error) + end + end, Subscriptions). -bin(L) -> iolist_to_binary(L). diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index f8ca04ea2..50d438f16 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -16,7 +16,8 @@ -export([all/0, init_per_suite/1, end_per_suite/1]). -export([t_rpc/1, - t_mqtt/1 + t_mqtt/1, + t_forwards_mngr/1 ]). -include_lib("eunit/include/eunit.hrl"). @@ -27,7 +28,8 @@ -define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). all() -> [t_rpc, - t_mqtt + t_mqtt, + t_forwards_mngr ]. init_per_suite(Config) -> @@ -43,6 +45,24 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). +t_forwards_mngr(Config) when is_list(Config) -> + Subs = [{<<"a">>, 1}, {<<"b">>, 2}], + Cfg = #{address => node(), + forwards => [<<"mngr">>], + connect_module => emqx_portal_rpc, + mountpoint => <<"forwarded">>, + subscriptions => Subs + }, + Name = ?FUNCTION_NAME, + {ok, Pid} = emqx_portal:start_link(Name, Cfg), + try + ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Name)), + ?assertEqual(Subs, emqx_portal:get_subscriptions(Pid)) + after + ok = emqx_portal:stop(Pid) + end. + + %% A loopback RPC to local node t_rpc(Config) when is_list(Config) -> Cfg = #{address => node(), From 2903a810cedb34bf9a7398c52aaa6b42f641b3b7 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 17 Feb 2019 11:24:14 +0100 Subject: [PATCH 07/26] Add emqx_portal:ensure_foreard_present API --- Makefile | 9 ++++++- src/portal/emqx_portal.erl | 48 ++++++++++++++++++++++++--------- test/emqx_ct_broker_helpers.erl | 15 ++++++++++- test/emqx_portal_SUITE.erl | 7 ++--- 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index f6230d2c7..5c0fd0bba 100644 --- a/Makefile +++ b/Makefile @@ -101,12 +101,19 @@ rebar-eunit: $(CUTTLEFISH_SCRIPT) rebar-compile: @rebar3 compile -rebar-ct: app.config +rebar-ct-setup: app.config @rebar3 as test compile @ln -s -f '../../../../etc' _build/test/lib/emqx/ @ln -s -f '../../../../data' _build/test/lib/emqx/ + +rebar-ct: rebar-ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') +## Run one single CT with rebar3 +## e.g. make ct-one-suite suite=emqx_portal +ct-one-suite: rebar-ct-setup + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE + rebar-clean: @rebar3 clean diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 238b6681f..a0f954e1a 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -73,7 +73,7 @@ -export([connecting/3, connected/3]). %% management APIs --export([get_forwards/1]). %, add_forward/2, del_forward/2]). +-export([get_forwards/1, ensure_forward_present/2]). %, del_forward/2]). -export([get_subscriptions/1]). %, add_subscription/3, del_subscription/2]). -export_type([config/0, @@ -86,6 +86,7 @@ -type config() :: map(). -type batch() :: [emqx_portal_msg:exp_msg()]. -type ack_ref() :: term(). +-type topic() :: emqx_topic:topic(). -include("logger.hrl"). -include("emqx_mqtt.hrl"). @@ -137,8 +138,13 @@ handle_ack(Pid, Ref) when node() =:= node(Pid) -> Pid ! {batch_ack, Ref}, ok. --spec get_forwards(id()) -> [emqx_topic:topic()]. -get_forwards(Id) -> gen_statem:call(id(Id), get_forwards). +%% @doc Return all forwards (local subscriptions). +-spec get_forwards(id()) -> [topic()]. +get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)). + +%% @doc Add a new forward (local topic subscription). +-spec ensure_forward_present(id(), topic()) -> ok | {error, any()}. +ensure_forward_present(Id, Topic) -> gen_statem:call(id(Id), {ensure_forward_present, topic(Topic)}). -spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). @@ -185,7 +191,7 @@ init(Config) -> batch_count_limit => GetQ(batch_count_limit, ?DEFAULT_BATCH_COUNT), max_inflight_batches => Get(max_inflight_batches, ?DEFAULT_SEND_AHEAD), mountpoint => format_mountpoint(Get(mountpoint, undefined)), - topics => Topics, + forwards => Topics, subscriptions => Subs, replayq => Queue, inflight => [] @@ -274,6 +280,16 @@ connected(Type, Content, State) -> %% Common handlers common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> {keep_state_and_data, [{reply, From, Forwards}]}; +common(_StateName, {call, From}, {ensure_forward_present, Topic}, + #{forwards := Forwards} = State) -> + case lists:member(Topic, Forwards) of + true -> + {keep_state_and_data, [{reply, From, ok}]}; + false -> + ok = subscribe_local_topic(Topic), + {keep_state, State#{forwards := lists:usort([Topic | Forwards])}, + [{reply, From, ok}]} + end; common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> {keep_state_and_data, [{reply, From, Subs}]}; common(_StateName, info, {dispatch, _, Msg}, @@ -281,8 +297,8 @@ common(_StateName, info, {dispatch, _, Msg}, NewQ = replayq:append(Q, collect([Msg])), {keep_state, State#{replayq => NewQ}, ?maybe_send}; common(StateName, Type, Content, State) -> - ?DEBUG("Portal ~p discarded ~p type event at state ~p:~p", - [name(), Type, StateName, Content]), + ?INFO("Portal ~p discarded ~p type event at state ~p:~p", + [name(), Type, StateName, Content]), {keep_state, State}. collect(Acc) -> @@ -346,13 +362,19 @@ do_ack(#{inflight := Inflight}, Ref) -> false -> stale end. -subscribe_local_topics(Topics) -> - lists:foreach( - fun(Topic0) -> - Topic = iolist_to_binary(Topic0), - emqx_topic:validate({filter, Topic}) orelse erlang:error({bad_topic, Topic}), - emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}) - end, Topics). +subscribe_local_topics(Topics) -> lists:foreach(fun subscribe_local_topic/1, Topics). + +subscribe_local_topic(Topic0) -> + Topic = topic(Topic0), + try + emqx_topic:validate({filter, Topic}) + catch + error : Reason -> + erlang:error({bad_topic, Topic, Reason}) + end, + ok = emqx_broker:subscribe(Topic, #{qos => ?QOS_1, subid => name()}). + +topic(T) -> iolist_to_binary(T). disconnect(#{connection := Conn, conn_ref := ConnRef, diff --git a/test/emqx_ct_broker_helpers.erl b/test/emqx_ct_broker_helpers.erl index 1ef5b1fa3..0e0bfa3a4 100644 --- a/test/emqx_ct_broker_helpers.erl +++ b/test/emqx_ct_broker_helpers.erl @@ -54,10 +54,17 @@ "ECDH-RSA-AES128-SHA","AES128-SHA"]}]). run_setup_steps() -> + _ = run_setup_steps([]), + %% return ok to be backward compatible + ok. + +run_setup_steps(Config) -> NewConfig = generate_config(), lists:foreach(fun set_app_env/1, NewConfig), set_bridge_env(), - application:ensure_all_started(?APP). + {ok, _} = application:ensure_all_started(?APP), + set_log_level(Config), + Config. run_teardown_steps() -> ?APP:shutdown(). @@ -67,6 +74,12 @@ generate_config() -> Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), cuttlefish_generator:map(Schema, Conf). +set_log_level(Config) -> + case proplists:get_value(log_level, Config) of + undefined -> ok; + Level -> emqx_logger:set_log_level(Level) + end. + get_base_dir(Module) -> {file, Here} = code:is_loaded(Module), filename:dirname(filename:dirname(Here)). diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 50d438f16..2c92afd94 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -39,8 +39,7 @@ init_per_suite(Config) -> _ -> ok end, - emqx_ct_broker_helpers:run_setup_steps(), - Config. + emqx_ct_broker_helpers:run_setup_steps(Config). end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). @@ -57,12 +56,14 @@ t_forwards_mngr(Config) when is_list(Config) -> {ok, Pid} = emqx_portal:start_link(Name, Cfg), try ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Name)), + ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr")), + ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr2")), + ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_portal:get_forwards(Pid)), ?assertEqual(Subs, emqx_portal:get_subscriptions(Pid)) after ok = emqx_portal:stop(Pid) end. - %% A loopback RPC to local node t_rpc(Config) when is_list(Config) -> Cfg = #{address => node(), From 599f5c8d4f8007faa157352e00118830345bd967 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 17 Feb 2019 11:38:44 +0100 Subject: [PATCH 08/26] Add API emqx_portal:ensure_forward_absent --- src/portal/emqx_portal.erl | 21 ++++++++++++++++++--- test/emqx_portal_SUITE.erl | 5 ++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index a0f954e1a..508fb81a9 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -73,7 +73,7 @@ -export([connecting/3, connected/3]). %% management APIs --export([get_forwards/1, ensure_forward_present/2]). %, del_forward/2]). +-export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). -export([get_subscriptions/1]). %, add_subscription/3, del_subscription/2]). -export_type([config/0, @@ -143,8 +143,13 @@ handle_ack(Pid, Ref) when node() =:= node(Pid) -> get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)). %% @doc Add a new forward (local topic subscription). --spec ensure_forward_present(id(), topic()) -> ok | {error, any()}. -ensure_forward_present(Id, Topic) -> gen_statem:call(id(Id), {ensure_forward_present, topic(Topic)}). +-spec ensure_forward_present(id(), topic()) -> ok. +ensure_forward_present(Id, Topic) -> + gen_statem:call(id(Id), {ensure_forward_present, topic(Topic)}). + +-spec ensure_forward_absent(id(), topic()) -> ok. +ensure_forward_absent(Id, Topic) -> + gen_statem:call(id(Id), {ensure_forward_absent, topic(Topic)}). -spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). @@ -290,6 +295,16 @@ common(_StateName, {call, From}, {ensure_forward_present, Topic}, {keep_state, State#{forwards := lists:usort([Topic | Forwards])}, [{reply, From, ok}]} end; +common(_StateName, {call, From}, {ensure_forward_absent, Topic}, + #{forwards := Forwards} = State) -> + case lists:member(Topic, Forwards) of + true -> + emqx_broker:unsubscribe(Topic), + {keep_state, State#{forwards := lists:delete(Topic, Forwards)}, + [{reply, From, ok}]}; + false -> + {keep_state_and_data, [{reply, From, ok}]} + end; common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> {keep_state_and_data, [{reply, From, Subs}]}; common(_StateName, info, {dispatch, _, Msg}, diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 2c92afd94..fb851c400 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -39,7 +39,7 @@ init_per_suite(Config) -> _ -> ok end, - emqx_ct_broker_helpers:run_setup_steps(Config). + emqx_ct_broker_helpers:run_setup_steps([{log_leve, info} | Config]). end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). @@ -59,6 +59,9 @@ t_forwards_mngr(Config) when is_list(Config) -> ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr")), ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr2")), ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_portal:get_forwards(Pid)), + ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr2")), + ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr3")), + ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Pid)), ?assertEqual(Subs, emqx_portal:get_subscriptions(Pid)) after ok = emqx_portal:stop(Pid) From 786a6eb696f7b85de2a70129433088d2eda70b15 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 17 Feb 2019 13:59:53 +0100 Subject: [PATCH 09/26] Add APIs for subscription add / delete --- src/emqx_portal_connect.erl | 8 ++- src/portal/emqx_portal.erl | 108 +++++++++++++++++++++++--------- src/portal/emqx_portal_mqtt.erl | 17 +++++ test/emqx_portal_SUITE.erl | 34 ++++++++-- 4 files changed, 131 insertions(+), 36 deletions(-) diff --git a/src/emqx_portal_connect.erl b/src/emqx_portal_connect.erl index ab3a3f5b8..79ec077e8 100644 --- a/src/emqx_portal_connect.erl +++ b/src/emqx_portal_connect.erl @@ -18,7 +18,7 @@ -export_type([config/0, connection/0]). --optional_callbacks([]). +-optional_callbacks([ensure_subscribed/3, ensure_unsubscribed/2]). %% map fields depend on implementation -type config() :: map(). @@ -26,6 +26,8 @@ -type conn_ref() :: term(). -type batch() :: emqx_protal:batch(). -type ack_ref() :: emqx_portal:ack_ref(). +-type topic() :: emqx_topic:topic(). +-type qos() :: emqx_mqtt_types:qos(). -include("logger.hrl"). @@ -42,6 +44,10 @@ %% called when owner is shutting down. -callback stop(conn_ref(), connection()) -> ok. +-callback ensure_subscribed(connection(), topic(), qos()) -> ok. + +-callback ensure_unsubscribed(connection(), topic()) -> ok. + start(Module, Config) -> case Module:start(Config) of {ok, Ref, Conn} -> diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 508fb81a9..eb5f86754 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -74,7 +74,7 @@ %% management APIs -export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). --export([get_subscriptions/1]). %, add_subscription/3, del_subscription/2]). +-export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]). -export_type([config/0, batch/0, @@ -142,17 +142,32 @@ handle_ack(Pid, Ref) when node() =:= node(Pid) -> -spec get_forwards(id()) -> [topic()]. get_forwards(Id) -> gen_statem:call(id(Id), get_forwards, timer:seconds(1000)). +%% @doc Return all subscriptions (subscription over mqtt connection to remote broker). +-spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. +get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). + %% @doc Add a new forward (local topic subscription). -spec ensure_forward_present(id(), topic()) -> ok. ensure_forward_present(Id, Topic) -> - gen_statem:call(id(Id), {ensure_forward_present, topic(Topic)}). + gen_statem:call(id(Id), {ensure_present, forwards, topic(Topic)}). +%% @doc Ensure a forward topic is deleted. -spec ensure_forward_absent(id(), topic()) -> ok. ensure_forward_absent(Id, Topic) -> - gen_statem:call(id(Id), {ensure_forward_absent, topic(Topic)}). + gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}). --spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. -get_subscriptions(Id) -> gen_statem:call(id(Id), get_subscriptions). +%% @doc Ensure subscribed to remote topic. +%% NOTE: only applicable when connection module is emqx_portal_mqtt +%% return `{error, no_remote_subscription_support}' otherwise. +-spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. +ensure_subscription_present(Id, Topic, QoS) -> + gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}). + +%% @doc Ensure unsubscribed from remote topic. +%% NOTE: only applicable when connection module is emqx_portal_mqtt +-spec ensure_subscription_absent(id(), topic()) -> ok. +ensure_subscription_absent(Id, Topic) -> + gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}). callback_mode() -> [state_functions, state_enter]. @@ -187,7 +202,7 @@ init(Config) -> mountpoint, forwards ], Config#{subscriptions => Subs}), - ConnectFun = fun() -> emqx_portal_connect:start(ConnectModule, ConnectConfig) end, + ConnectFun = fun(SubsX) -> emqx_portal_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end, {ok, connecting, #{connect_module => ConnectModule, connect_fun => ConnectFun, @@ -217,8 +232,10 @@ connecting(enter, connected, #{reconnect_delay_ms := Timeout}) -> Action = {state_timeout, Timeout, reconnect}, {keep_state_and_data, Action}; connecting(enter, connecting, #{reconnect_delay_ms := Timeout, - connect_fun := ConnectFun} = State) -> - case ConnectFun() of + connect_fun := ConnectFun, + subscriptions := Subs + } = State) -> + case ConnectFun(Subs) of {ok, ConnRef, Conn} -> Action = {state_timeout, 0, connected}, {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; @@ -277,7 +294,7 @@ connected(info, {batch_ack, Ref}, State) -> %% try re-connect then re-send {next_state, connecting, disconnect(State)}; {ok, NewState} -> - {keep_state, NewState} + {keep_state, NewState, ?maybe_send} end; connected(Type, Content, State) -> common(connected, Type, Content, State). @@ -285,28 +302,14 @@ connected(Type, Content, State) -> %% Common handlers common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> {keep_state_and_data, [{reply, From, Forwards}]}; -common(_StateName, {call, From}, {ensure_forward_present, Topic}, - #{forwards := Forwards} = State) -> - case lists:member(Topic, Forwards) of - true -> - {keep_state_and_data, [{reply, From, ok}]}; - false -> - ok = subscribe_local_topic(Topic), - {keep_state, State#{forwards := lists:usort([Topic | Forwards])}, - [{reply, From, ok}]} - end; -common(_StateName, {call, From}, {ensure_forward_absent, Topic}, - #{forwards := Forwards} = State) -> - case lists:member(Topic, Forwards) of - true -> - emqx_broker:unsubscribe(Topic), - {keep_state, State#{forwards := lists:delete(Topic, Forwards)}, - [{reply, From, ok}]}; - false -> - {keep_state_and_data, [{reply, From, ok}]} - end; common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> {keep_state_and_data, [{reply, From, Subs}]}; +common(_StateName, {call, From}, {ensure_present, What, Topic}, State) -> + {Result, NewState} = ensure_present(What, Topic, State), + {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> + {Result, NewState} = ensure_absent(What, Topic, State), + {keep_state, NewState, [{reply, From, Result}]}; common(_StateName, info, {dispatch, _, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, collect([Msg])), @@ -316,6 +319,53 @@ common(StateName, Type, Content, State) -> [name(), Type, StateName, Content]), {keep_state, State}. +ensure_present(Key, Topic, State) -> + Topics = maps:get(Key, State), + case is_topic_present(Topic, Topics) of + true -> + {ok, State}; + false -> + R = do_ensure_present(Key, Topic, State), + {R, State#{Key := lists:usort([Topic | Topics])}} + end. + +ensure_absent(Key, Topic, State) -> + Topics = maps:get(Key, State), + case is_topic_present(Topic, Topics) of + true -> + R = do_ensure_absent(Key, Topic, State), + {R, State#{Key := ensure_topic_absent(Topic, Topics)}}; + false -> + {ok, State} + end. + +ensure_topic_absent(_Topic, []) -> []; +ensure_topic_absent(Topic, [{_, _} | _] = L) -> lists:keydelete(Topic, 1, L); +ensure_topic_absent(Topic, L) -> lists:delete(Topic, L). + +is_topic_present({Topic, _QoS}, Topics) -> + is_topic_present(Topic, Topics); +is_topic_present(Topic, Topics) -> + lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics). + +do_ensure_present(forwards, Topic, _) -> + ok = subscribe_local_topic(Topic); +do_ensure_present(subscriptions, {Topic, QoS}, + #{connect_module := ConnectModule, connection := Conn}) -> + case erlang:function_exported(ConnectModule, ensure_subscribed, 3) of + true -> ConnectModule:ensure_subscribed(Conn, Topic, QoS); + false -> {error, no_remote_subscription_support} + end. + +do_ensure_absent(forwards, Topic, _) -> + ok = emqx_broker:unsubscribe(Topic); +do_ensure_absent(subscriptions, Topic, #{connect_module := ConnectModule, + connection := Conn}) -> + case erlang:function_exported(ConnectModule, ensure_unsubscribed, 2) of + true -> ConnectModule:ensure_unsubscribed(Conn, Topic); + false -> {error, no_remote_subscription_support} + end. + collect(Acc) -> receive {dispatch, _, Msg} -> diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index ba7461943..0817e3fe2 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -23,6 +23,11 @@ stop/2 ]). +%% optional behaviour callbacks +-export([ensure_subscribed/3, + ensure_unsubscribed/2 + ]). + -include("emqx_mqtt.hrl"). -define(ACK_REF(ClientPid, PktId), {ClientPid, PktId}). @@ -66,6 +71,18 @@ stop(Ref, #{ack_collector := AckCollector, client_pid := Pid}) -> safe_stop(Pid, fun() -> emqx_client:stop(Pid) end, 1000), ok. +ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) -> + emqx_client:subscribe(Pid, Topic, QoS); +ensure_subscribed(_Conn, _Topic, _QoS) -> + %% return ok for now, next re-connect should should call start with new topic added to config + ok. + +ensure_unsubscribed(#{client_pid := Pid}, Topic) when is_pid(Pid) -> + emqx_client:unsubscribe(Pid, Topic); +ensure_unsubscribed(_, _) -> + %% return ok for now, next re-connect should should call start with this topic deleted from config + ok. + safe_stop(Pid, StopF, Timeout) -> MRef = monitor(process, Pid), unlink(Pid), diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index fb851c400..c375ee994 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -17,7 +17,7 @@ -export([all/0, init_per_suite/1, end_per_suite/1]). -export([t_rpc/1, t_mqtt/1, - t_forwards_mngr/1 + t_mngr/1 ]). -include_lib("eunit/include/eunit.hrl"). @@ -29,7 +29,7 @@ all() -> [t_rpc, t_mqtt, - t_forwards_mngr + t_mngr ]. init_per_suite(Config) -> @@ -44,7 +44,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). -t_forwards_mngr(Config) when is_list(Config) -> +t_mngr(Config) when is_list(Config) -> Subs = [{<<"a">>, 1}, {<<"b">>, 2}], Cfg = #{address => node(), forwards => [<<"mngr">>], @@ -62,6 +62,10 @@ t_forwards_mngr(Config) when is_list(Config) -> ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr2")), ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr3")), ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Pid)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_portal:ensure_subscription_present(Pid, <<"t">>, 0)), + ?assertEqual({error, no_remote_subscription_support}, + emqx_portal:ensure_subscription_absent(Pid, <<"t">>)), ?assertEqual(Subs, emqx_portal:get_subscriptions(Pid)) after ok = emqx_portal:stop(Pid) @@ -93,10 +97,16 @@ t_rpc(Config) when is_list(Config) -> ok = emqx_portal:stop(Pid) end. +%% Full data loopback flow explained: +%% test-pid ---> mock-cleint ----> local-broker ---(local-subscription)---> +%% portal(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> +%% portal(import) --(mecked message sending)--> test-pid t_mqtt(Config) when is_list(Config) -> SendToTopic = <<"t_mqtt/one">>, + SendToTopic2 = <<"t_mqtt/two">>, Mountpoint = <<"forwarded/${node}/">>, ForwardedTopic = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic]), + ForwardedTopic2 = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic2]), Cfg = #{address => "127.0.0.1:1883", forwards => [SendToTopic], connect_module => emqx_portal_mqtt, @@ -118,7 +128,7 @@ t_mqtt(Config) when is_list(Config) -> start_type => manual, %% Consume back to forwarded message for verification %% NOTE: this is a indefenite loopback without mocking emqx_portal:import_batch/2 - subscriptions => [{ForwardedTopic, 1}] + subscriptions => [{ForwardedTopic, _QoS = 1}] }, Tester = self(), Ref = make_ref(), @@ -131,15 +141,27 @@ t_mqtt(Config) when is_list(Config) -> {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), ClientId = <<"client-1">>, try + ?assertEqual([{ForwardedTopic, 1}], emqx_portal:get_subscriptions(Pid)), + emqx_portal:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), + ?assertEqual([{ForwardedTopic, 1}, + {ForwardedTopic2, 1}], emqx_portal:get_subscriptions(Pid)), {ok, ConnPid} = emqx_mock_client:start_link(ClientId), {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), %% message from a different client, to avoid getting terminated by no-local - Msgs = lists:seq(1, 10), + Max = 100, + Msgs = lists:seq(1, Max), lists:foreach(fun(I) -> Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic, integer_to_binary(I)), emqx_session:publish(SPid, I, Msg) end, Msgs), ok = receive_and_match_messages(Ref, Msgs), + ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), + Msgs2 = lists:seq(Max + 1, Max * 2), + lists:foreach(fun(I) -> + Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic2, integer_to_binary(I)), + emqx_session:publish(SPid, I, Msg) + end, Msgs2), + ok = receive_and_match_messages(Ref, Msgs2), emqx_mock_client:close_session(ConnPid) after ok = emqx_portal:stop(Pid), @@ -147,7 +169,7 @@ t_mqtt(Config) when is_list(Config) -> end. receive_and_match_messages(Ref, Msgs) -> - TRef = erlang:send_after(timer:seconds(4), self(), {Ref, timeout}), + TRef = erlang:send_after(timer:seconds(5), self(), {Ref, timeout}), try do_receive_and_match_messages(Ref, Msgs) after From d7e18c95c69e6ade69b99195a7e835e2d655de66 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Tue, 19 Feb 2019 10:10:05 +0800 Subject: [PATCH 10/26] Fix spelling error --- etc/emqx.conf | 4 ++-- src/emqx_portal_connect.erl | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 554238f2c..889ad4357 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1695,7 +1695,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## bridge.aws.subscription.2.qos = 1 ## Maximum number of messages in one batch when sending to remote borkers -## NOTE: when bridging viar MQTT connection to remote broker, this config is only +## NOTE: when bridging via MQTT connection to remote broker, this config is only ## used for internal message passing optimization as the underlying MQTT ## protocol does not supports batching. ## @@ -1848,7 +1848,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## bridge.azure.subscription.2.qos = 1 ## Maximum number of messages in one batch when sending to remote borkers -## NOTE: when bridging viar MQTT connection to remote broker, this config is only +## NOTE: when bridging via MQTT connection to remote broker, this config is only ## used for internal message passing optimization as the underlying MQTT ## protocol does not supports batching. ## diff --git a/src/emqx_portal_connect.erl b/src/emqx_portal_connect.erl index 79ec077e8..f0b7474a5 100644 --- a/src/emqx_portal_connect.erl +++ b/src/emqx_portal_connect.erl @@ -67,6 +67,5 @@ obfuscate(Map) -> end end, [], Map). -is_sensitive(passsword) -> true; +is_sensitive(password) -> true; is_sensitive(_) -> false. - From 6e1d4ec2616f62a906fd7173cfdd9750d4fd2c68 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Tue, 19 Feb 2019 22:40:46 +0100 Subject: [PATCH 11/26] Add emqx_portal SUITE to default CT list --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5c0fd0bba..9aa90f94c 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_keepalive emqx_lib emqx_metrics emqx_mod emqx_mod_sup emqx_mqtt_caps \ emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ - emqx_listeners emqx_protocol emqx_pool emqx_shared_sub \ + emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_portal \ emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message From efc9e340333cc3fffb201c356bc851a75a80dbba Mon Sep 17 00:00:00 2001 From: spring2maz Date: Tue, 19 Feb 2019 22:41:43 +0100 Subject: [PATCH 12/26] Make use of BUMP_PACKET_ID the only way to generate packet IDs --- Makefile | 2 ++ include/emqx_mqtt.hrl | 5 ----- src/emqx_client.erl | 22 +++++++++++++++++----- src/portal/emqx_portal_mqtt.erl | 4 ++-- test/emqx_portal_SUITE.erl | 8 ++++++-- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 9aa90f94c..c4e0dc020 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,8 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message +CT_SUITES = emqx_portal + CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 3bba42216..1c2ce1a27 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -176,11 +176,6 @@ -define(MAX_PACKET_ID, 16#ffff). -define(MAX_PACKET_SIZE, 16#fffffff). --define(BUMP_PACKET_ID(Base, Bump), - case Base + Bump of - __I__ when __I__ > ?MAX_PACKET_ID -> __I__ - ?MAX_PACKET_ID; - __I__ -> __I__ - end). %%-------------------------------------------------------------------- %% MQTT Frame Mask diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 71527871f..fbcfcc059 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -35,6 +35,7 @@ -export([pubcomp/2, pubcomp/3, pubcomp/4]). -export([subscriptions/1]). -export([info/1, stop/1]). +-export([next_packet_id/1, next_packet_id/2]). %% For test cases -export([pause/1, resume/1]). @@ -421,6 +422,19 @@ disconnect(Client, ReasonCode) -> disconnect(Client, ReasonCode, Properties) -> gen_statem:call(Client, {disconnect, ReasonCode, Properties}). +-spec next_packet_id(packet_id()) -> packet_id(). +next_packet_id(?MAX_PACKET_ID) -> 1; +next_packet_id(Id) -> Id + 1. + +-spec next_packet_id(packet_id(), integer()) -> packet_id(). +next_packet_id(Id, Bump) -> + true = (Bump < ?MAX_PACKET_ID div 2), %% assert + N = Id + Bump, + case N > ?MAX_PACKET_ID of + true -> N - ?MAX_PACKET_ID; + false -> N + end. + %%------------------------------------------------------------------------------ %% For test cases %%------------------------------------------------------------------------------ @@ -1354,7 +1368,7 @@ send(Packet, State = #state{socket = Sock, proto_ver = Ver}) Data = emqx_frame:serialize(Packet, #{version => Ver}), emqx_logger:debug("SEND Data: ~p", [Data]), case emqx_client_sock:send(Sock, Data) of - ok -> {ok, next_packet_id(State)}; + ok -> {ok, bump_last_packet_id(State)}; Error -> Error end. @@ -1397,8 +1411,6 @@ assign_packet_id([H | T], Id) -> assign_packet_id([], _Id) -> []. -next_packet_id(State = #state{last_packet_id = Id}) -> - State#state{last_packet_id = next_packet_id(Id)}; -next_packet_id(16#ffff) -> 1; -next_packet_id(Id) -> Id + 1. +bump_last_packet_id(State = #state{last_packet_id = Id}) -> + State#state{last_packet_id = next_packet_id(Id)}. diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index 0817e3fe2..de5bf6042 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -103,7 +103,7 @@ safe_stop(Pid, StopF, Timeout) -> send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, Batch) -> case emqx_client:publish(ClientPid, Batch) of {ok, BasePktId} -> - LastPktId = ?BUMP_PACKET_ID(BasePktId, length(Batch) - 1), + LastPktId = emqx_client:next_packet_id(BasePktId, length(Batch) - 1), AckCollector ! ?SENT(?RANGE(BasePktId, LastPktId)), %% return last pakcet id as batch reference {ok, LastPktId}; @@ -145,7 +145,7 @@ match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, PktId) | Sent]) -> ok = emqx_portal:handle_ack(Parent, PktId), match_acks(Parent, Acked, Sent); match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, Max) | Sent]) -> - match_acks(Parent, Acked, [?RANGE(PktId + 1, Max) | Sent]). + match_acks(Parent, Acked, [?RANGE(emqx_client:next_packet_id(PktId), Max) | Sent]). %% When puback for QoS-1 message is received from remote MQTT broker %% NOTE: no support for QoS-2 diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index c375ee994..96b11fcf9 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -156,6 +156,7 @@ t_mqtt(Config) when is_list(Config) -> end, Msgs), ok = receive_and_match_messages(Ref, Msgs), ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), + timer:sleep(200), Msgs2 = lists:seq(Max + 1, Max * 2), lists:foreach(fun(I) -> Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic2, integer_to_binary(I)), @@ -181,8 +182,11 @@ do_receive_and_match_messages(_Ref, []) -> ok; do_receive_and_match_messages(Ref, [I | Rest]) -> receive {Ref, timeout} -> erlang:error(timeout); - {Ref, [#{payload := P}]} -> - ?assertEqual(I, binary_to_integer(P)), + {Ref, [#{payload := P} = Msg]} -> + case I =:= binary_to_integer(P) of + true -> ok; + false -> throw({unexpected, Msg, [I | Rest]}) + end, do_receive_and_match_messages(Ref, Rest) end. From 086a1d56b97a8b5cd0875983eb8b81a90db40ddb Mon Sep 17 00:00:00 2001 From: spring2maz Date: Tue, 19 Feb 2019 22:43:10 +0100 Subject: [PATCH 13/26] Drop unused config schema bridge.$name.transport was added before we decided to derive transport portocol based on the 'address' config. i.e. when it's a remote erlang node, use gen_rpc otherwise (must be IP or hostnmae), we should estabilish mqtt connection --- priv/emqx.schema | 5 ----- src/portal/emqx_portal.erl | 7 +++++-- test/emqx_portal_SUITE.erl | 7 ++++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index 7082d4f87..b2623128a 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1512,11 +1512,6 @@ end}. %%-------------------------------------------------------------------- %% Bridges %%-------------------------------------------------------------------- -{mapping, "bridge.$name.transport", "emqx.bridges", [ - {default, mqtt_client}, - {datatype, {enum, [emqx_portal, mqtt_client]}} -]}. - {mapping, "bridge.$name.address", "emqx.bridges", [ {datatype, string} ]}. diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index eb5f86754..3673731f6 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -353,8 +353,11 @@ do_ensure_present(forwards, Topic, _) -> do_ensure_present(subscriptions, {Topic, QoS}, #{connect_module := ConnectModule, connection := Conn}) -> case erlang:function_exported(ConnectModule, ensure_subscribed, 3) of - true -> ConnectModule:ensure_subscribed(Conn, Topic, QoS); - false -> {error, no_remote_subscription_support} + true -> + _ = ConnectModule:ensure_subscribed(Conn, Topic, QoS), + ok; + false -> + {error, no_remote_subscription_support} end. do_ensure_absent(forwards, Topic, _) -> diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 96b11fcf9..f42a9e7b1 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -142,7 +142,10 @@ t_mqtt(Config) when is_list(Config) -> ClientId = <<"client-1">>, try ?assertEqual([{ForwardedTopic, 1}], emqx_portal:get_subscriptions(Pid)), - emqx_portal:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), + ok = emqx_portal:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), + ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), + %% TODO: investigate why it's necessary + timer:sleep(1000), ?assertEqual([{ForwardedTopic, 1}, {ForwardedTopic2, 1}], emqx_portal:get_subscriptions(Pid)), {ok, ConnPid} = emqx_mock_client:start_link(ClientId), @@ -155,8 +158,6 @@ t_mqtt(Config) when is_list(Config) -> emqx_session:publish(SPid, I, Msg) end, Msgs), ok = receive_and_match_messages(Ref, Msgs), - ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), - timer:sleep(200), Msgs2 = lists:seq(Max + 1, Max * 2), lists:foreach(fun(I) -> Msg = emqx_message:make(<<"client-2">>, ?QOS_1, SendToTopic2, integer_to_binary(I)), From 1626cade28f7c2f9b58f8dacf01f8d00f2481f11 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Wed, 20 Feb 2019 12:10:47 +0100 Subject: [PATCH 14/26] Deleted batch publish support in emqx_portal_client eqmx_portal_mqtt has to do single message publish calls for now Also fix a bug in emqx_portal_mqtt ack collector --- Makefile | 2 - src/emqx_client.erl | 79 ++++++++++----------------------- src/portal/emqx_portal_mqtt.erl | 40 ++++++++++------- test/emqx_portal_SUITE.erl | 2 +- test/emqx_portal_mqtt_tests.erl | 37 ++++----------- test/emqx_portal_tests.erl | 4 +- 6 files changed, 58 insertions(+), 106 deletions(-) diff --git a/Makefile b/Makefile index c4e0dc020..9aa90f94c 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,6 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message -CT_SUITES = emqx_portal - CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) diff --git a/src/emqx_client.erl b/src/emqx_client.erl index fbcfcc059..48c8c0e1e 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -35,7 +35,6 @@ -export([pubcomp/2, pubcomp/3, pubcomp/4]). -export([subscriptions/1]). -export([info/1, stop/1]). --export([next_packet_id/1, next_packet_id/2]). %% For test cases -export([pause/1, resume/1]). @@ -390,7 +389,7 @@ publish(Client, Topic, Properties, Payload, Opts) props = Properties, payload = iolist_to_binary(Payload)}). --spec(publish(client(), #mqtt_msg{} | [#mqtt_msg{}]) -> ok | {ok, packet_id()} | {error, term()}). +-spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}). publish(Client, Msg) -> gen_statem:call(Client, {publish, Msg}). @@ -422,19 +421,6 @@ disconnect(Client, ReasonCode) -> disconnect(Client, ReasonCode, Properties) -> gen_statem:call(Client, {disconnect, ReasonCode, Properties}). --spec next_packet_id(packet_id()) -> packet_id(). -next_packet_id(?MAX_PACKET_ID) -> 1; -next_packet_id(Id) -> Id + 1. - --spec next_packet_id(packet_id(), integer()) -> packet_id(). -next_packet_id(Id, Bump) -> - true = (Bump < ?MAX_PACKET_ID div 2), %% assert - N = Id + Bump, - case N > ?MAX_PACKET_ID of - true -> N - ?MAX_PACKET_ID; - false -> N - end. - %%------------------------------------------------------------------------------ %% For test cases %%------------------------------------------------------------------------------ @@ -801,26 +787,23 @@ connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) -> {stop_and_reply, Reason, [{reply, From, Error}]} end; -connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, State) - when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> - connected({call, From}, {publish, [Msg]}, State); - -%% when publishing a batch, {ok, BasePacketId} is returned, -%% following packet ids for the batch tail are mod (1 bsl 16) consecutive -connected({call, From}, {publish, Msgs}, - State = #state{inflight = Inflight, last_packet_id = PacketId}) when is_list(Msgs) -> - %% NOTE: to ensure API call atomicity, inflight buffer may overflow - case emqx_inflight:is_full(Inflight) of - true -> - {keep_state, State, [{reply, From, {error, inflight_full}}]}; - false -> - case send_batch(assign_packet_id(Msgs, PacketId), State) of - {ok, NewState} -> - {keep_state, ensure_retry_timer(NewState), [{reply, From, {ok, PacketId}}]}; - {error, Reason} -> - {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} - end - end; +connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, + State = #state{inflight = Inflight, last_packet_id = PacketId}) + when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> + case emqx_inflight:is_full(Inflight) of + true -> + {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]}; + false -> + Msg1 = Msg#mqtt_msg{packet_id = PacketId}, + case send(Msg1, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), + {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), + [{reply, From, {ok, PacketId}}]}; + {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} + end + end; connected({call, From}, UnsubReq = {unsubscribe, Properties, Topics}, State = #state{last_packet_id = PacketId}) -> @@ -1349,24 +1332,13 @@ send_puback(Packet, State) -> {error, Reason} -> {stop, {shutdown, Reason}} end. -send_batch([], State) -> {ok, State}; -send_batch([Msg = #mqtt_msg{packet_id = PacketId} | Rest], - State = #state{inflight = Inflight}) -> - case send(Msg, State) of - {ok, NewState} -> - Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg, os:timestamp()}, Inflight), - send_batch(Rest, NewState#state{inflight = Inflight1}); - {error, Reason} -> - {error, Reason} - end. - send(Msg, State) when is_record(Msg, mqtt_msg) -> send(msg_to_packet(Msg), State); send(Packet, State = #state{socket = Sock, proto_ver = Ver}) when is_record(Packet, mqtt_packet) -> Data = emqx_frame:serialize(Packet, #{version => Ver}), - emqx_logger:debug("SEND Data: ~p", [Data]), + emqx_logger:debug("SEND Data: ~1000p", [Packet]), case emqx_client_sock:send(Sock, Data) of ok -> {ok, bump_last_packet_id(State)}; Error -> Error @@ -1402,15 +1374,12 @@ next_events(Packets) -> [{next_event, cast, Packet} || Packet <- lists:reverse(Packets)]. %%------------------------------------------------------------------------------ -%% packet_id generation and assignment - -assign_packet_id(Msg = #mqtt_msg{}, Id) -> - Msg#mqtt_msg{packet_id = Id}; -assign_packet_id([H | T], Id) -> - [assign_packet_id(H, Id) | assign_packet_id(T, next_packet_id(Id))]; -assign_packet_id([], _Id) -> - []. +%% packet_id generation bump_last_packet_id(State = #state{last_packet_id = Id}) -> State#state{last_packet_id = next_packet_id(Id)}. +-spec next_packet_id(packet_id()) -> packet_id(). +next_packet_id(?MAX_PACKET_ID) -> 1; +next_packet_id(Id) -> Id + 1. + diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index de5bf6042..6c718e9e4 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -34,7 +34,8 @@ %% Messages towards ack collector process -define(RANGE(Min, Max), {Min, Max}). --define(SENT(PktIdRange), {sent, PktIdRange}). +-define(REF_IDS(Ref, Ids), {Ref, Ids}). +-define(SENT(RefIds), {sent, RefIds}). -define(ACKED(AnyPktId), {acked, AnyPktId}). -define(STOP(Ref), {stop, Ref}). @@ -49,8 +50,6 @@ start(Config) -> {ok, _} -> try subscribe_remote_topics(Pid, maps:get(subscriptions, Config, [])), - %% ack collector is always a new pid every reconnect. - %% use it as a connection reference {ok, Ref, #{ack_collector => AckCollector, client_pid => Pid}} catch @@ -100,16 +99,21 @@ safe_stop(Pid, StopF, Timeout) -> exit(Pid, kill) end. -send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, Batch) -> - case emqx_client:publish(ClientPid, Batch) of - {ok, BasePktId} -> - LastPktId = emqx_client:next_packet_id(BasePktId, length(Batch) - 1), - AckCollector ! ?SENT(?RANGE(BasePktId, LastPktId)), - %% return last pakcet id as batch reference - {ok, LastPktId}; +send(Conn, Batch) -> + send(Conn, Batch, []). + +send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, [Msg | Rest] = Batch, Acc) -> + case emqx_client:publish(ClientPid, Msg) of + {ok, PktId} when Rest =:= [] -> + %% last one sent + Ref = make_ref(), + AckCollector ! ?SENT(?REF_IDS(Ref, lists:reverse([PktId | Acc]))), + {ok, Ref}; + {ok, PktId} -> + send(Conn, Rest, [PktId | Acc]); {error, {_PacketId, inflight_full}} -> timer:sleep(100), - send(Conn, Batch); + send(Conn, Batch, Acc); {error, Reason} -> %% NOTE: There is no partial sucess of a batch and recover from the middle %% only to retry all messages in one batch @@ -126,9 +130,9 @@ ack_collector(Parent, ConnRef, Acked, Sent) -> exit(normal); ?ACKED(PktId) -> match_acks(Parent, queue:in(PktId, Acked), Sent); - ?SENT(Range) -> + ?SENT(RefIds) -> %% this message only happens per-batch, hence ++ is ok - match_acks(Parent, Acked, Sent ++ [Range]) + match_acks(Parent, Acked, Sent ++ [RefIds]) after 200 -> {Acked, Sent} @@ -140,12 +144,14 @@ match_acks(Parent, Acked, Sent) -> match_acks_1(Parent, queue:out(Acked), Sent). match_acks_1(_Parent, {empty, Empty}, Sent) -> {Empty, Sent}; -match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, PktId) | Sent]) -> +match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId]) | Sent]) -> %% batch finished - ok = emqx_portal:handle_ack(Parent, PktId), + ok = emqx_portal:handle_ack(Parent, Ref), match_acks(Parent, Acked, Sent); -match_acks_1(Parent, {{value, PktId}, Acked}, [?RANGE(PktId, Max) | Sent]) -> - match_acks(Parent, Acked, [?RANGE(emqx_client:next_packet_id(PktId), Max) | Sent]). +match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId | RestIds]) | Sent]) -> + %% one message finished, but not the whole batch + match_acks(Parent, Acked, [?REF_IDS(Ref, RestIds) | Sent]). + %% When puback for QoS-1 message is received from remote MQTT broker %% NOTE: no support for QoS-2 diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index f42a9e7b1..3c380d684 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -39,7 +39,7 @@ init_per_suite(Config) -> _ -> ok end, - emqx_ct_broker_helpers:run_setup_steps([{log_leve, info} | Config]). + emqx_ct_broker_helpers:run_setup_steps([{log_level, error} | Config]). end_per_suite(_Config) -> emqx_ct_broker_helpers:run_teardown_steps(). diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_portal_mqtt_tests.erl index 8f513a853..311554a2f 100644 --- a/test/emqx_portal_mqtt_tests.erl +++ b/test/emqx_portal_mqtt_tests.erl @@ -18,7 +18,6 @@ send_and_ack_test() -> %% delegate from gen_rpc to rpc for unit test - Tester = self(), meck:new(emqx_client, [passthrough, no_history]), meck:expect(emqx_client, start_link, 1, fun(#{msg_handler := Hdlr}) -> @@ -28,14 +27,13 @@ send_and_ack_test() -> meck:expect(emqx_client, stop, 1, fun(Pid) -> Pid ! stop end), meck:expect(emqx_client, publish, 2, - fun(_Conn, Msgs) -> - case rand:uniform(100) of + fun(Client, Msg) -> + case rand:uniform(200) of 1 -> {error, {dummy, inflight_full}}; _ -> - BaseId = hd(Msgs), - Tester ! {published, Msgs}, - {ok, BaseId} + Client ! {publish, Msg}, + {ok, Msg} %% as packet id end end), try @@ -44,38 +42,19 @@ send_and_ack_test() -> {ok, Ref, Conn} = emqx_portal_mqtt:start(#{}), %% return last packet id as batch reference {ok, AckRef} = emqx_portal_mqtt:send(Conn, Batch), - %% as if the remote broker replied with puback - ok = fake_pubacks(Conn), %% expect batch ack - AckRef1= receive {batch_ack, Id} -> Id end, - %% asset received ack matches the batch ref returned in send API - ?assertEqual(AckRef, AckRef1), + receive {batch_ack, AckRef} -> ok end, ok = emqx_portal_mqtt:stop(Ref, Conn) after meck:unload(emqx_client) end. -fake_pubacks(#{client_pid := Client}) -> - #{puback := PubAckCallback} = get_hdlr(Client), +fake_client(#{puback := PubAckCallback} = Hdlr) -> receive - {published, Msgs} -> - lists:foreach( - fun(Id) -> - PubAckCallback(#{packet_id => Id, reason_code => ?RC_SUCCESS}) - end, Msgs) - end. - -get_hdlr(Client) -> - Client ! {get_hdlr, self()}, - receive {hdr, Hdlr} -> Hdlr end. - -fake_client(Hdlr) -> - receive - {get_hdlr, Pid} -> - Pid ! {hdr, Hdlr}, + {publish, PktId} -> + PubAckCallback(#{packet_id => PktId, reason_code => ?RC_SUCCESS}), fake_client(Hdlr); stop -> exit(normal) end. - diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index 3b2879b12..b44e02732 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -71,7 +71,7 @@ disturbance_test() -> %% buffer should continue taking in messages when disconnected buffer_when_disconnected_test_() -> - {timeout, 5000, fun test_buffer_when_disconnected/0}. + {timeout, 10000, fun test_buffer_when_disconnected/0}. test_buffer_when_disconnected() -> Ref = make_ref(), @@ -92,7 +92,7 @@ test_buffer_when_disconnected() -> Receiver ! {portal, Pid}, ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), Pid ! {disconnected, Ref, test}, - ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 2000), + ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000), ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), ok = emqx_portal:stop(?PORTAL_REG_NAME). From 796fc3b1ba548a7f90cb074a53b5dc1d5d80cf9d Mon Sep 17 00:00:00 2001 From: Gilbert Date: Thu, 21 Feb 2019 13:41:14 +0800 Subject: [PATCH 15/26] Fix app config generation (#2245) --- etc/emqx.conf | 270 +++++++++++++++++++------------------ priv/emqx.schema | 8 +- test/emqx_portal_SUITE.erl | 1 - 3 files changed, 145 insertions(+), 134 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 889ad4357..ebe1bdd1c 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1596,34 +1596,12 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ##-------------------------------------------------------------------- ## Bridges to aws ##-------------------------------------------------------------------- -## Start type of the bridge. -## -## Value: enum -## manual -## auto -## bridge.aws.start_type = manual - -## Bridge reconnect time. -## -## Value: Duration -## Default: 30 seconds -## bridge.aws.reconnect_interval = 30s - -## Retry interval for bridge QoS1 message delivering. -## -## Value: Duration -## bridge.aws.retry_interval = 20s - -## Inflight size. -## -## Value: Integer -## bridge.aws.max_inflight = 32 ## Bridge address: node name for local bridge, host:port for remote. ## ## Value: String ## Example: emqx@127.0.0.1, 127.0.0.1:1883 -## bridge.aws.address = 127.0.0.1:1883 +bridge.aws.address = 127.0.0.1:1883 ## Protocol version of the bridge. ## @@ -1631,12 +1609,12 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## - mqttv5 ## - mqttv4 ## - mqttv3 -## bridge.aws.proto_ver = mqttv4 +bridge.aws.proto_ver = mqttv4 ## The ClientId of a remote bridge. ## ## Value: String -## bridge.aws.client_id = bridge_aws +bridge.aws.client_id = bridge_aws ## The Clean start flag of a remote bridge. ## @@ -1645,54 +1623,107 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## ## NOTE: Some IoT platforms require clean_start ## must be set to 'true' -## bridge.aws.clean_start = true +bridge.aws.clean_start = true ## The username for a remote bridge. ## ## Value: String -## bridge.aws.username = user +bridge.aws.username = user ## The password for a remote bridge. ## ## Value: String -## bridge.aws.password = passwd +bridge.aws.password = passwd ## Mountpoint of the bridge. ## ## Value: String -## bridge.aws.mountpoint = bridge/aws/${node}/ - -## Ping interval of a down bridge. -## -## Value: Duration -## Default: 10 seconds -## bridge.aws.keepalive = 60s +bridge.aws.mountpoint = bridge/aws/${node}/ ## Forward message topics ## ## Value: String ## Example: topic1/#,topic2/# -## bridge.aws.forwards = topic1/#,topic2/# +bridge.aws.forwards = topic1/#,topic2/# + +## Bribge to remote server via SSL. +## +## Value: on | off +bridge.aws.ssl = off + +## PEM-encoded CA certificates of the bridge. +## +## Value: File +bridge.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem + +## Client SSL Certfile of the bridge. +## +## Value: File +bridge.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem + +## Client SSL Keyfile of the bridge. +## +## Value: File +bridge.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem + +## SSL Ciphers used by the bridge. +## +## Value: String +bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +bridge.aws.keepalive = 60s + +## TLS versions used by the bridge. +## +## Value: String +bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.aws.subscription.1.topic = cmd/topic1 +bridge.aws.subscription.1.topic = cmd/topic1 ## Subscriptions of the bridge qos. ## ## Value: Number -## bridge.aws.subscription.1.qos = 1 +bridge.aws.subscription.1.qos = 1 ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.aws.subscription.2.topic = cmd/topic2 +bridge.aws.subscription.2.topic = cmd/topic2 ## Subscriptions of the bridge qos. ## ## Value: Number -## bridge.aws.subscription.2.qos = 1 +bridge.aws.subscription.2.qos = 1 + +## Start type of the bridge. +## +## Value: enum +## manual +## auto +bridge.aws.start_type = manual + +## Bridge reconnect time. +## +## Value: Duration +## Default: 30 seconds +bridge.aws.reconnect_interval = 30s + +## Retry interval for bridge QoS1 message delivering. +## +## Value: Duration +bridge.aws.retry_interval = 20s + +## Inflight size. +## +## Value: Integer +bridge.aws.max_inflight = 32 ## Maximum number of messages in one batch when sending to remote borkers ## NOTE: when bridging via MQTT connection to remote broker, this config is only @@ -1701,76 +1732,23 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## ## Value: Integer ## default: 32 -## bridge.aws.queue.batch_size = 32 +bridge.aws.queue.batch_size = 32 ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, ## replayq works in a mem-only manner. ## ## Value: String -## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ +bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ ## Replayq segment size ## ## Value: Bytesize - -## bridge.aws.queue.replayq_seg_bytes = 10MB - -## Bribge to remote server via SSL. -## -## Value: on | off -## bridge.aws.ssl = off - -## PEM-encoded CA certificates of the bridge. -## -## Value: File -## bridge.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem - -## Client SSL Certfile of the bridge. -## -## Value: File -## bridge.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem - -## Client SSL Keyfile of the bridge. -## -## Value: File -## bridge.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem - -## SSL Ciphers used by the bridge. -## -## Value: String -## bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 - -## TLS versions used by the bridge. -## -## Value: String -## bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +bridge.aws.queue.replayq_seg_bytes = 10MB ##-------------------------------------------------------------------- ## Bridges to azure ##-------------------------------------------------------------------- -## Start type of the bridge. -## -## Value: enum -## manual -## auto -## bridge.azure.start_type = manual - -## Bridge reconnect count. -## -## Value: Number -## bridge.azure.reconnect_count = 10 - -## Bridge reconnect time. -## -## Value: Duration -## Default: 30 seconds -## bridge.azure.reconnect_time = 30s - -## Retry interval for bridge QoS1 message delivering. -## -## Value: Duration -## bridge.azure.retry_interval = 20s ## Bridge address: node name for local bridge, host:port for remote. ## @@ -1789,7 +1767,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## The ClientId of a remote bridge. ## ## Value: String -## bridge.azure.client_id = bridge_azure +## bridge.azure.client_id = bridge_aws ## The Clean start flag of a remote bridge. ## @@ -1813,13 +1791,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Mountpoint of the bridge. ## ## Value: String -## bridge.azure.mountpoint = bridge/azure/${node}/ - -## Ping interval of a down bridge. -## -## Value: Duration -## Default: 10 seconds -## bridge.azure.keepalive = 10s +## bridge.azure.mountpoint = bridge/aws/${node}/ ## Forward message topics ## @@ -1827,10 +1799,46 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Example: topic1/#,topic2/# ## bridge.azure.forwards = topic1/#,topic2/# +## Bribge to remote server via SSL. +## +## Value: on | off +## bridge.azure.ssl = off + +## PEM-encoded CA certificates of the bridge. +## +## Value: File +## bridge.azure.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem + +## Client SSL Certfile of the bridge. +## +## Value: File +## bridge.azure.certfile = {{ platform_etc_dir }}/certs/client-cert.pem + +## Client SSL Keyfile of the bridge. +## +## Value: File +## bridge.azure.keyfile = {{ platform_etc_dir }}/certs/client-key.pem + +## SSL Ciphers used by the bridge. +## +## Value: String +## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +## bridge.azure.keepalive = 60s + +## TLS versions used by the bridge. +## +## Value: String +## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1 + ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.azure.subscription.1.topic = $share/cmd/topic1 +## bridge.azure.subscription.1.topic = cmd/topic1 ## Subscriptions of the bridge qos. ## @@ -1840,13 +1848,36 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Subscriptions of the bridge topic. ## ## Value: String -## bridge.azure.subscription.2.topic = $share/cmd/topic2 +## bridge.azure.subscription.2.topic = cmd/topic2 ## Subscriptions of the bridge qos. ## ## Value: Number ## bridge.azure.subscription.2.qos = 1 +## Start type of the bridge. +## +## Value: enum +## manual +## auto +## bridge.azure.start_type = manual + +## Bridge reconnect time. +## +## Value: Duration +## Default: 30 seconds +## bridge.azure.reconnect_interval = 30s + +## Retry interval for bridge QoS1 message delivering. +## +## Value: Duration +## bridge.azure.retry_interval = 20s + +## Inflight size. +## +## Value: Integer +## bridge.azure.max_inflight = 32 + ## Maximum number of messages in one batch when sending to remote borkers ## NOTE: when bridging via MQTT connection to remote broker, this config is only ## used for internal message passing optimization as the underlying MQTT @@ -1861,38 +1892,13 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## replayq works in a mem-only manner. ## ## Value: String -## Default: "" -## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_azure.bridge/ +## bridge.azure.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ ## Replayq segment size ## ## Value: Bytesize ## bridge.azure.queue.replayq_seg_bytes = 10MB -## PEM-encoded CA certificates of the bridge. -## -## Value: File -## bridge.azure.cacertfile = cacert.pem - -## Client SSL Certfile of the bridge. -## -## Value: File -## bridge.azure.certfile = cert.pem - -## Client SSL Keyfile of the bridge. -## -## Value: File -## bridge.azure.keyfile = key.pem - -## SSL Ciphers used by the bridge. -## -## Value: String -## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 - -## TLS versions used by the bridge. -## -## Value: String -## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1 ##-------------------------------------------------------------------- ## Modules diff --git a/priv/emqx.schema b/priv/emqx.schema index b2623128a..388b9beea 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1595,6 +1595,11 @@ end}. {datatype, {duration, ms}} ]}. +{mapping, "bridge.$name.retry_interval", "emqx.bridges", [ + {default, "20s"}, + {datatype, {duration, ms}} +]}. + {mapping, "bridge.$name.max_inflight", "emqx.bridges", [ {default, 0}, {datatype, integer} @@ -1659,7 +1664,8 @@ end}. end end, ConnMod = fun(Name) -> - [Addr] = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".address", Conf), + [AddrConfig] = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".address", Conf), + {_, Addr} = AddrConfig, Subs = Subscriptions(Name), case IsNodeAddr(Addr) of true when Subs =/= [] -> diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 3c380d684..8b80fb72f 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -190,4 +190,3 @@ do_receive_and_match_messages(Ref, [I | Rest]) -> end, do_receive_and_match_messages(Ref, Rest) end. - From d4495fd8e72ee0667410bd665d9f310a9278de90 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Sun, 24 Feb 2019 21:55:03 +0100 Subject: [PATCH 16/26] Add manual start API --- src/portal/emqx_portal.erl | 82 +++++++++++++++++++++++++++++++++----- test/emqx_portal_SUITE.erl | 10 +++-- test/emqx_portal_tests.erl | 17 +++++++- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 3673731f6..76f7c1842 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -36,11 +36,12 @@ %% %% Batch collector state diagram %% -%% [connecting] --(2)--> [connected] -%% | ^ | -%% | | | -%% '--(1)---'--------(3)------' +%% [standing_by] --(0) --> [connecting] --(2)--> [connected] +%% | ^ | +%% | | | +%% '--(1)---'--------(3)------' %% +%% (0): auto or manual start %% (1): retry timeout %% (2): successfuly connected to remote node/cluster %% (3): received {disconnected, conn_ref(), Reason} OR @@ -70,9 +71,10 @@ -export([terminate/3, code_change/4, init/1, callback_mode/0]). %% state functions --export([connecting/3, connected/3]). +-export([standing_by/3, connecting/3, connected/3]). %% management APIs +-export([ensure_started/2, ensure_stopped/1, ensure_stopped/2]). -export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). -export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]). @@ -100,6 +102,8 @@ -define(maybe_send, {next_event, internal, maybe_send}). %% @doc Start a portal worker. Supported configs: +%% start_type: 'manual' (default) or 'auto', when manual, portal will stay +%% at 'standing_by' state until a manual call to start it. %% connect_module: The module which implements emqx_portal_connect behaviour %% and work as message batch transport layer %% reconnect_delay_ms: Delay in milli-seconds for the portal worker to retry @@ -123,6 +127,38 @@ start_link(Name, Config) when is_list(Config) -> start_link(Name, Config) -> gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []). +%% @doc Manually start portal worker. State idempotency ensured. +ensure_started(Name, Config) -> + case start_link(Name, Config) of + {ok, Pid} -> {ok, Pid}; + {error, {already_started,Pid}} -> {ok, Pid} + end. + +%% @doc Manually stop portal worker. State idempotency ensured. +ensure_stopped(Id) -> + ensure_stopped(Id, 1000). + +ensure_stopped(Id, Timeout) -> + Pid = case id(Id) of + P when is_pid(P) -> P; + N -> whereis(N) + end, + case Pid of + undefined -> + ok; + _ -> + MRef = monitor(process, Pid), + unlink(Pid), + _ = gen_statem:call(id(Id), ensure_stopped, Timeout), + receive + {'DOWN', MRef, _, _, _} -> + ok + after + Timeout -> + exit(Pid, kill) + end + end. + stop(Pid) -> gen_statem:stop(Pid). %% @doc This function is to be evaluated on message/batch receiver side. @@ -193,7 +229,6 @@ init(Config) -> true = emqx_topic:validate({filter, T}), {T, QoS} end, Get(subscriptions, []))), - ok = subscribe_local_topics(Topics), ConnectModule = maps:get(connect_module, Config), ConnectConfig = maps:without([connect_module, queue, @@ -203,9 +238,10 @@ init(Config) -> forwards ], Config#{subscriptions => Subs}), ConnectFun = fun(SubsX) -> emqx_portal_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end, - {ok, connecting, + {ok, standing_by, #{connect_module => ConnectModule, connect_fun => ConnectFun, + start_type => Get(start_type, manual), reconnect_delay_ms => maps:get(reconnect_delay_ms, Config, ?DEFAULT_RECONNECT_DELAY_MS), batch_bytes_limit => GetQ(batch_bytes_limit, ?DEFAULT_BATCH_BYTES), batch_count_limit => GetQ(batch_count_limit, ?DEFAULT_BATCH_COUNT), @@ -225,16 +261,36 @@ terminate(_Reason, _StateName, #{replayq := Q} = State) -> _ = replayq:close(Q), ok. +%% @doc Standing by for manual start. +standing_by(enter, _, #{start_type := auto}) -> + Action = {state_timeout, 0, do_connect}, + {keep_state_and_data, Action}; +standing_by(enter, _, #{start_type := manual}) -> + keep_state_and_data; +standing_by({call, From}, ensure_started, State) -> + {next_state, connecting, State, [{reply, From, ok}]}; +standing_by({call, From}, ensure_stopped, _State) -> + {stop_and_reply, {shutdown, manual}, [{reply, From, ok}]}; +standing_by(state_timeout, do_connect, State) -> + {next_state, connecting, State}; +standing_by({call, From}, _Call, _State) -> + {keep_state_and_data, [{reply, From, {error, standing_by}}]}; +standing_by(info, Info, State) -> + ?INFO("Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), + {keep_state_and_data, State}. + %% @doc Connecting state is a state with timeout. %% After each timeout, it re-enters this state and start a retry until %% successfuly connected to remote node/cluster. connecting(enter, connected, #{reconnect_delay_ms := Timeout}) -> Action = {state_timeout, Timeout, reconnect}, {keep_state_and_data, Action}; -connecting(enter, connecting, #{reconnect_delay_ms := Timeout, - connect_fun := ConnectFun, - subscriptions := Subs - } = State) -> +connecting(enter, _, #{reconnect_delay_ms := Timeout, + connect_fun := ConnectFun, + subscriptions := Subs, + forwards := Forwards + } = State) -> + ok = subscribe_local_topics(Forwards), case ConnectFun(Subs) of {ok, ConnRef, Conn} -> Action = {state_timeout, 0, connected}, @@ -300,6 +356,8 @@ connected(Type, Content, State) -> common(connected, Type, Content, State). %% Common handlers +common(_StateName, {call, From}, ensure_started, _State) -> + {keep_state_and_data, [{reply, From, ok}]}; common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> {keep_state_and_data, [{reply, From, Forwards}]}; common(_StateName, {call, From}, get_subscriptions, #{subscriptions := Subs}) -> @@ -310,6 +368,8 @@ common(_StateName, {call, From}, {ensure_present, What, Topic}, State) -> common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> {Result, NewState} = ensure_absent(What, Topic, State), {keep_state, NewState, [{reply, From, Result}]}; +common(_StateName, {call, From}, ensure_stopped, _State) -> + {stop_and_reply, {shutdown, manual}, [{reply, From, ok}]}; common(_StateName, info, {dispatch, _, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, collect([Msg])), diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 8b80fb72f..3d34eda50 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -50,7 +50,8 @@ t_mngr(Config) when is_list(Config) -> forwards => [<<"mngr">>], connect_module => emqx_portal_rpc, mountpoint => <<"forwarded">>, - subscriptions => Subs + subscriptions => Subs, + start_type => auto }, Name = ?FUNCTION_NAME, {ok, Pid} = emqx_portal:start_link(Name, Cfg), @@ -76,7 +77,8 @@ t_rpc(Config) when is_list(Config) -> Cfg = #{address => node(), forwards => [<<"t_rpc/#">>], connect_module => emqx_portal_rpc, - mountpoint => <<"forwarded">> + mountpoint => <<"forwarded">>, + start_type => auto }, {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), ClientId = <<"ClientId">>, @@ -125,10 +127,10 @@ t_mqtt(Config) when is_list(Config) -> }, reconnect_delay_ms => 1000, ssl => false, - start_type => manual, %% Consume back to forwarded message for verification %% NOTE: this is a indefenite loopback without mocking emqx_portal:import_batch/2 - subscriptions => [{ForwardedTopic, _QoS = 1}] + subscriptions => [{ForwardedTopic, _QoS = 1}], + start_type => auto }, Tester = self(), Ref = make_ref(), diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index b44e02732..03f545b38 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -96,6 +96,20 @@ test_buffer_when_disconnected() -> ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), ok = emqx_portal:stop(?PORTAL_REG_NAME). +manual_start_stop_test() -> + Ref = make_ref(), + Config0 = make_config(Ref, self(), {ok, Ref, connection}), + Config = Config0#{start_type := manual}, + {ok, Pid} = emqx_portal:ensure_started(?PORTAL_NAME, Config), + %% call ensure_started again should yeld the same result + {ok, Pid} = emqx_portal:ensure_started(?PORTAL_NAME, Config), + ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), + ?assertEqual({error, standing_by}, + emqx_portal:ensure_forward_present(Pid, "dummy")), + ok = emqx_portal:ensure_stopped(unknown), + ok = emqx_portal:ensure_stopped(Pid), + ok = emqx_portal:ensure_stopped(?PORTAL_REG_NAME). + %% Feed messages to portal sender_loop(_Pid, [], _) -> exit(normal); sender_loop(Pid, [Num | Rest], Interval) -> @@ -133,7 +147,8 @@ make_config(Ref, TestPid, Result) -> test_ref => Ref, connect_module => ?MODULE, reconnect_delay_ms => 50, - connect_result => Result + connect_result => Result, + start_type => auto }. make_msg(I) -> From 921a45a5057d5a75fcaf827a035214efb18ed0f0 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Mon, 25 Feb 2019 09:20:05 +0800 Subject: [PATCH 17/26] Fix emqx_portal_mqtt_tests start function --- src/emqx_client.erl | 1 - src/portal/emqx_portal_mqtt.erl | 15 +++++++++++---- test/emqx_portal_mqtt_tests.erl | 3 +-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 48c8c0e1e..987cf52d3 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -1382,4 +1382,3 @@ bump_last_packet_id(State = #state{last_packet_id = Id}) -> -spec next_packet_id(packet_id()) -> packet_id(). next_packet_id(?MAX_PACKET_ID) -> 1; next_packet_id(Id) -> Id + 1. - diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index 6c718e9e4..f47a0cf5a 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -39,12 +39,20 @@ -define(ACKED(AnyPktId), {acked, AnyPktId}). -define(STOP(Ref), {stop, Ref}). -start(Config) -> +start(Config = #{address := Address}) -> Ref = make_ref(), Parent = self(), AckCollector = spawn_link(fun() -> ack_collector(Parent, Ref) end), Handlers = make_hdlr(Parent, AckCollector, Ref), - case emqx_client:start_link(Config#{msg_handler => Handlers, owner => AckCollector}) of + {Host, Port} = case string:tokens(Address, ":") of + [H] -> {H, 1883}; + [H, P] -> {H, list_to_integer(P)} + end, + ClientConfig = Config#{msg_handler => Handlers, + owner => AckCollector, + host => Host, + port => Port}, + case emqx_client:start_link(ClientConfig) of {ok, Pid} -> case emqx_client:connect(Pid) of {ok, _} -> @@ -58,7 +66,7 @@ start(Config) -> {error, Reason} end; {error, Reason} -> - ok = stop(AckCollector, Pid), + ok = stop(Ref, #{ack_collector => AckCollector, client_pid => Pid}), {error, Reason} end; {error, Reason} -> @@ -178,4 +186,3 @@ subscribe_remote_topics(ClientPid, Subscriptions) -> Error -> throw(Error) end end, Subscriptions). - diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_portal_mqtt_tests.erl index 311554a2f..788f5a4b2 100644 --- a/test/emqx_portal_mqtt_tests.erl +++ b/test/emqx_portal_mqtt_tests.erl @@ -39,7 +39,7 @@ send_and_ack_test() -> try Max = 100, Batch = lists:seq(1, Max), - {ok, Ref, Conn} = emqx_portal_mqtt:start(#{}), + {ok, Ref, Conn} = emqx_portal_mqtt:start(#{address => "127.0.0.1:1883"}), %% return last packet id as batch reference {ok, AckRef} = emqx_portal_mqtt:send(Conn, Batch), %% expect batch ack @@ -57,4 +57,3 @@ fake_client(#{puback := PubAckCallback} = Hdlr) -> stop -> exit(normal) end. - From ec37225333b2ad71ee82c033eb0fae0fb90c1214 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Mon, 25 Feb 2019 16:59:10 +0800 Subject: [PATCH 18/26] Add emqx_portal interfaces --- .gitignore | 1 + src/portal/emqx_portal.erl | 36 +++++++++++++++++++++++++++------- src/portal/emqx_portal_sup.erl | 5 ++++- test/emqx_portal_tests.erl | 7 +++---- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 7a4e891d1..ab0cbe156 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ cuttlefish rebar.lock xrefr erlang.mk +*.coverdata diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 76f7c1842..66d83eacc 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -74,6 +74,7 @@ -export([standing_by/3, connecting/3, connected/3]). %% management APIs +-export([start_bridge/1, stop_bridge/1, status/1]). -export([ensure_started/2, ensure_stopped/1, ensure_stopped/2]). -export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). -export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]). @@ -161,6 +162,15 @@ ensure_stopped(Id, Timeout) -> stop(Pid) -> gen_statem:stop(Pid). +start_bridge(Name) -> + gen_statem:call(name(Name), ensure_started). + +stop_bridge(Name) -> + gen_statem:call(name(Name), ensure_stopped). + +status(Pid) -> + gen_statem:call(Pid, status). + %% @doc This function is to be evaluated on message/batch receiver side. -spec import_batch(batch(), fun(() -> ok)) -> ok. import_batch(Batch, AckFun) -> @@ -268,16 +278,21 @@ standing_by(enter, _, #{start_type := auto}) -> standing_by(enter, _, #{start_type := manual}) -> keep_state_and_data; standing_by({call, From}, ensure_started, State) -> - {next_state, connecting, State, [{reply, From, ok}]}; + {next_state, connecting, State, + [{reply, From, <<"starting bridge ......">>}]}; standing_by({call, From}, ensure_stopped, _State) -> - {stop_and_reply, {shutdown, manual}, [{reply, From, ok}]}; + {keep_state_and_data, [{reply, From, <<"bridge not started">>}]}; +standing_by({call, From}, status, _State) -> + {keep_state_and_data, [{reply, From, <<"Stopped">>}]}; standing_by(state_timeout, do_connect, State) -> {next_state, connecting, State}; standing_by({call, From}, _Call, _State) -> {keep_state_and_data, [{reply, From, {error, standing_by}}]}; standing_by(info, Info, State) -> ?INFO("Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), - {keep_state_and_data, State}. + {keep_state_and_data, State}; +standing_by(Type, Content, State) -> + common(connecting, Type, Content, State). %% @doc Connecting state is a state with timeout. %% After each timeout, it re-enters this state and start a retry until @@ -303,6 +318,10 @@ connecting(state_timeout, connected, State) -> {next_state, connected, State}; connecting(state_timeout, reconnect, _State) -> repeat_state_and_data; +connecting({call, From}, status, _State) -> + {keep_state_and_data, [{reply, From, <<"Stopped">>}]}; +connecting({call, From}, _Call, _State) -> + {keep_state_and_data, [{reply, From, <<"starting bridge ......">>}]}; connecting(info, {batch_ack, Ref}, State) -> case do_ack(State, Ref) of {ok, NewState} -> @@ -334,14 +353,17 @@ connected(internal, maybe_send, State) -> {error, NewState} -> {next_state, connecting, disconnect(NewState)} end; +connected({call, From}, ensure_started, _State) -> + {keep_state_and_data, [{reply, From, <<"bridge already started">>}]}; +connected({call, From}, status, _State) -> + {keep_state_and_data, [{reply, From, <<"Running">>}]}; connected(info, {disconnected, ConnRef, Reason}, #{conn_ref := ConnRef, connection := Conn} = State) -> ?INFO("Portal ~p diconnected~nreason=~p", [name(), Conn, Reason]), {next_state, connecting, State#{conn_ref := undefined, - connection := undefined - }}; + connection := undefined}}; connected(info, {batch_ack, Ref}, State) -> case do_ack(State, Ref) of stale -> @@ -369,7 +391,8 @@ common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> {Result, NewState} = ensure_absent(What, Topic, State), {keep_state, NewState, [{reply, From, Result}]}; common(_StateName, {call, From}, ensure_stopped, _State) -> - {stop_and_reply, {shutdown, manual}, [{reply, From, ok}]}; + {stop_and_reply, {shutdown, manual}, + [{reply, From, <<"stop bridge successfully">>}]}; common(_StateName, info, {dispatch, _, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, collect([Msg])), @@ -536,4 +559,3 @@ name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). id(Pid) when is_pid(Pid) -> Pid; id(Name) -> name(Name). - diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl index 79afd6352..3f78f7680 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/portal/emqx_portal_sup.erl @@ -15,7 +15,7 @@ -module(emqx_portal_sup). -behavior(supervisor). --export([start_link/0, start_link/1]). +-export([start_link/0, start_link/1, portals/0]). -export([init/1]). @@ -52,3 +52,6 @@ portal_spec({Name, Config}) -> modules => [emqx_portal] }. +-spec(portals() -> [{node(), map()}]). +portals() -> + [{Name, emqx_portal:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?WORKER_SUP)]. diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index 03f545b38..e9a3583fe 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -106,9 +106,9 @@ manual_start_stop_test() -> ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), ?assertEqual({error, standing_by}, emqx_portal:ensure_forward_present(Pid, "dummy")), - ok = emqx_portal:ensure_stopped(unknown), - ok = emqx_portal:ensure_stopped(Pid), - ok = emqx_portal:ensure_stopped(?PORTAL_REG_NAME). + emqx_portal:ensure_stopped(unknown), + emqx_portal:ensure_stopped(Pid), + emqx_portal:ensure_stopped(?PORTAL_REG_NAME). %% Feed messages to portal sender_loop(_Pid, [], _) -> exit(normal); @@ -154,4 +154,3 @@ make_config(Ref, TestPid, Result) -> make_msg(I) -> Payload = integer_to_binary(I), emqx_message:make(<<"test/topic">>, Payload). - From ae92acf30f14c549302e9bae02ec724fa7689f72 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Mon, 25 Feb 2019 17:49:41 +0800 Subject: [PATCH 19/26] Refactor portal supervisor --- src/portal/emqx_portal_sup.erl | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl index 3f78f7680..19bb4e684 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/portal/emqx_portal_sup.erl @@ -28,20 +28,12 @@ start_link(Name) -> supervisor:start_link({local, Name}, ?MODULE, Name). init(?SUP) -> - Sp = fun(Name) -> - #{id => Name, - start => {?MODULE, start_link, [Name]}, - restart => permanent, - shutdown => 5000, - type => supervisor, - modules => [?MODULE] - } - end, - {ok, {{one_for_one, 5, 10}, [Sp(?WORKER_SUP)]}}; -init(?WORKER_SUP) -> BridgesConf = emqx_config:get_env(bridges, []), - BridgesSpec = lists:map(fun portal_spec/1, BridgesConf), - {ok, {{one_for_one, 10, 100}, BridgesSpec}}. + BridgeSpec = lists:map(fun portal_spec/1, BridgesConf), + SupFlag = #{strategy => one_for_one, + intensity => 10, + period => 100}, + {ok, {SupFlag, BridgeSpec}}. portal_spec({Name, Config}) -> #{id => Name, @@ -49,9 +41,8 @@ portal_spec({Name, Config}) -> restart => permanent, shutdown => 5000, type => worker, - modules => [emqx_portal] - }. + modules => [emqx_portal]}. -spec(portals() -> [{node(), map()}]). portals() -> - [{Name, emqx_portal:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?WORKER_SUP)]. + [{Name, emqx_portal:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. From afa0d98b8d44b195eb49d33e83b7b52bb73f8ed3 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Tue, 26 Feb 2019 10:21:28 +0800 Subject: [PATCH 20/26] Disable bridge defaultly --- etc/emqx.conf | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index ebe1bdd1c..8dc510a31 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1601,7 +1601,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## ## Value: String ## Example: emqx@127.0.0.1, 127.0.0.1:1883 -bridge.aws.address = 127.0.0.1:1883 +## bridge.aws.address = 127.0.0.1:1883 ## Protocol version of the bridge. ## @@ -1609,12 +1609,12 @@ bridge.aws.address = 127.0.0.1:1883 ## - mqttv5 ## - mqttv4 ## - mqttv3 -bridge.aws.proto_ver = mqttv4 +## bridge.aws.proto_ver = mqttv4 ## The ClientId of a remote bridge. ## ## Value: String -bridge.aws.client_id = bridge_aws +## bridge.aws.client_id = bridge_aws ## The Clean start flag of a remote bridge. ## @@ -1623,107 +1623,107 @@ bridge.aws.client_id = bridge_aws ## ## NOTE: Some IoT platforms require clean_start ## must be set to 'true' -bridge.aws.clean_start = true +## bridge.aws.clean_start = true ## The username for a remote bridge. ## ## Value: String -bridge.aws.username = user +## bridge.aws.username = user ## The password for a remote bridge. ## ## Value: String -bridge.aws.password = passwd +## bridge.aws.password = passwd ## Mountpoint of the bridge. ## ## Value: String -bridge.aws.mountpoint = bridge/aws/${node}/ +## bridge.aws.mountpoint = bridge/aws/${node}/ ## Forward message topics ## ## Value: String ## Example: topic1/#,topic2/# -bridge.aws.forwards = topic1/#,topic2/# +## bridge.aws.forwards = topic1/#,topic2/# ## Bribge to remote server via SSL. ## ## Value: on | off -bridge.aws.ssl = off +## bridge.aws.ssl = off ## PEM-encoded CA certificates of the bridge. ## ## Value: File -bridge.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## bridge.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem ## Client SSL Certfile of the bridge. ## ## Value: File -bridge.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem +## bridge.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem ## Client SSL Keyfile of the bridge. ## ## Value: File -bridge.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +## bridge.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem ## SSL Ciphers used by the bridge. ## ## Value: String -bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 +## bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 ## Ping interval of a down bridge. ## ## Value: Duration ## Default: 10 seconds -bridge.aws.keepalive = 60s +## bridge.aws.keepalive = 60s ## TLS versions used by the bridge. ## ## Value: String -bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 ## Subscriptions of the bridge topic. ## ## Value: String -bridge.aws.subscription.1.topic = cmd/topic1 +## bridge.aws.subscription.1.topic = cmd/topic1 ## Subscriptions of the bridge qos. ## ## Value: Number -bridge.aws.subscription.1.qos = 1 +## bridge.aws.subscription.1.qos = 1 ## Subscriptions of the bridge topic. ## ## Value: String -bridge.aws.subscription.2.topic = cmd/topic2 +## bridge.aws.subscription.2.topic = cmd/topic2 ## Subscriptions of the bridge qos. ## ## Value: Number -bridge.aws.subscription.2.qos = 1 +## bridge.aws.subscription.2.qos = 1 ## Start type of the bridge. ## ## Value: enum ## manual ## auto -bridge.aws.start_type = manual +## bridge.aws.start_type = manual ## Bridge reconnect time. ## ## Value: Duration ## Default: 30 seconds -bridge.aws.reconnect_interval = 30s +## bridge.aws.reconnect_interval = 30s ## Retry interval for bridge QoS1 message delivering. ## ## Value: Duration -bridge.aws.retry_interval = 20s +## bridge.aws.retry_interval = 20s ## Inflight size. ## ## Value: Integer -bridge.aws.max_inflight = 32 +## bridge.aws.max_inflight = 32 ## Maximum number of messages in one batch when sending to remote borkers ## NOTE: when bridging via MQTT connection to remote broker, this config is only @@ -1732,19 +1732,19 @@ bridge.aws.max_inflight = 32 ## ## Value: Integer ## default: 32 -bridge.aws.queue.batch_size = 32 +## bridge.aws.queue.batch_size = 32 ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, ## replayq works in a mem-only manner. ## ## Value: String -bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ +## bridge.aws.queue.replayq_dir = {{ platform_data_dir }}/emqx_aws_bridge/ ## Replayq segment size ## ## Value: Bytesize -bridge.aws.queue.replayq_seg_bytes = 10MB +## bridge.aws.queue.replayq_seg_bytes = 10MB ##-------------------------------------------------------------------- ## Bridges to azure From 75163e21f3bd598171ad38cfa356d7fc924c0663 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Tue, 26 Feb 2019 11:19:44 +0800 Subject: [PATCH 21/26] Reverse intensity/period in bridge SupFlag --- src/portal/emqx_portal.erl | 2 +- src/portal/emqx_portal_sup.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 66d83eacc..ff481f603 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -292,7 +292,7 @@ standing_by(info, Info, State) -> ?INFO("Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), {keep_state_and_data, State}; standing_by(Type, Content, State) -> - common(connecting, Type, Content, State). + common(standing_by, Type, Content, State). %% @doc Connecting state is a state with timeout. %% After each timeout, it re-enters this state and start a retry until diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl index 19bb4e684..fbc292c4e 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/portal/emqx_portal_sup.erl @@ -31,8 +31,8 @@ init(?SUP) -> BridgesConf = emqx_config:get_env(bridges, []), BridgeSpec = lists:map(fun portal_spec/1, BridgesConf), SupFlag = #{strategy => one_for_one, - intensity => 10, - period => 100}, + intensity => 100, + period => 10}, {ok, {SupFlag, BridgeSpec}}. portal_spec({Name, Config}) -> From 9dbc34c376e159963a66d3d1edec85d74713b7e9 Mon Sep 17 00:00:00 2001 From: spring2maz Date: Tue, 26 Feb 2019 08:45:15 +0100 Subject: [PATCH 22/26] Ack replayq and allow retry in tests --- src/emqx_client.erl | 23 ++++++++--- src/portal/emqx_portal.erl | 73 +++++++++++++++------------------ src/portal/emqx_portal_mqtt.erl | 6 +-- src/portal/emqx_portal_sup.erl | 14 ++++++- test/emqx_portal_SUITE.erl | 17 ++++---- test/emqx_portal_tests.erl | 3 +- 6 files changed, 77 insertions(+), 59 deletions(-) diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 987cf52d3..2b551be56 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -957,8 +957,7 @@ connected(cast, ?PACKET(?PINGRESP), State) -> end; connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), State) -> - ok = eval_msg_handler(State, disconnected, {ReasonCode, Properties}), - {stop, disconnected, State}; + {stop, {disconnected, ReasonCode, Properties}, State}; connected(info, {timeout, _TRef, keepalive}, State = #state{force_ping = true}) -> case send(?PACKET(?PINGREQ), State) of @@ -1042,10 +1041,18 @@ handle_event(EventType, EventContent, StateName, StateData) -> {keep_state, StateData}. %% Mandatory callback functions -terminate(_Reason, _State, #state{socket = undefined}) -> - ok; -terminate(_Reason, _State, #state{socket = Socket}) -> - emqx_client_sock:close(Socket). +terminate(Reason, _StateName, State = #state{socket = Socket}) -> + case Reason of + {disconnected, ReasonCode, Properties} -> + %% backward compatible + ok = eval_msg_handler(State, disconnected, {ReasonCode, Properties}); + _ -> + ok = eval_msg_handler(State, disconnected, Reason) + end, + case Socket =:= undefined of + true -> ok; + _ -> emqx_client_sock:close(Socket) + end. code_change(_Vsn, State, Data, _Extra) -> {ok, State, Data}. @@ -1276,6 +1283,10 @@ eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, %% Special handling for disconnected message when there is no handler callback Owner ! {disconnected, ReasonCode, Properties}, ok; +eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER}, + disconnected, _OtherReason) -> + %% do nothing to be backward compatible + ok; eval_msg_handler(#state{msg_handler = ?NO_REQ_HANDLER, owner = Owner}, Kind, Msg) -> Owner ! {Kind, Msg}, diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index ff481f603..83a29521d 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -64,8 +64,7 @@ -export([start_link/2, import_batch/2, handle_ack/2, - stop/1 - ]). + stop/1]). %% gen_statem callbacks -export([terminate/3, code_change/4, init/1, callback_mode/0]). @@ -74,15 +73,13 @@ -export([standing_by/3, connecting/3, connected/3]). %% management APIs --export([start_bridge/1, stop_bridge/1, status/1]). --export([ensure_started/2, ensure_stopped/1, ensure_stopped/2]). +-export([ensure_started/1, ensure_started/2, ensure_stopped/1, ensure_stopped/2, status/1]). -export([get_forwards/1, ensure_forward_present/2, ensure_forward_absent/2]). -export([get_subscriptions/1, ensure_subscription_present/3, ensure_subscription_absent/2]). -export_type([config/0, batch/0, - ack_ref/0 - ]). + ack_ref/0]). -type id() :: atom() | string() | pid(). -type qos() :: emqx_mqtt_types:qos(). @@ -112,7 +109,7 @@ %% max_inflight_batches: Max number of batches allowed to send-ahead before %% receiving confirmation from remote node/cluster %% mountpoint: The topic mount point for messages sent to remote node/cluster -%% `undefined', `<<>>' or `""' to disalble +%% `undefined', `<<>>' or `""' to disable %% forwards: Local topics to subscribe. %% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each %% send call towards emqx_portal_connect @@ -129,6 +126,9 @@ start_link(Name, Config) -> gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []). %% @doc Manually start portal worker. State idempotency ensured. +ensure_started(Name) -> + gen_statem:call(name(Name), ensure_started). + ensure_started(Name, Config) -> case start_link(Name, Config) of {ok, Pid} -> {ok, Pid}; @@ -162,12 +162,6 @@ ensure_stopped(Id, Timeout) -> stop(Pid) -> gen_statem:stop(Pid). -start_bridge(Name) -> - gen_statem:call(name(Name), ensure_started). - -stop_bridge(Name) -> - gen_statem:call(name(Name), ensure_stopped). - status(Pid) -> gen_statem:call(Pid, status). @@ -279,15 +273,11 @@ standing_by(enter, _, #{start_type := manual}) -> keep_state_and_data; standing_by({call, From}, ensure_started, State) -> {next_state, connecting, State, - [{reply, From, <<"starting bridge ......">>}]}; -standing_by({call, From}, ensure_stopped, _State) -> - {keep_state_and_data, [{reply, From, <<"bridge not started">>}]}; -standing_by({call, From}, status, _State) -> - {keep_state_and_data, [{reply, From, <<"Stopped">>}]}; + [{reply, From, ok}]}; standing_by(state_timeout, do_connect, State) -> {next_state, connecting, State}; standing_by({call, From}, _Call, _State) -> - {keep_state_and_data, [{reply, From, {error, standing_by}}]}; + {keep_state_and_data, [{reply, From, {error,standing_by}}]}; standing_by(info, Info, State) -> ?INFO("Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), {keep_state_and_data, State}; @@ -308,6 +298,7 @@ connecting(enter, _, #{reconnect_delay_ms := Timeout, ok = subscribe_local_topics(Forwards), case ConnectFun(Subs) of {ok, ConnRef, Conn} -> + ?INFO("Portal ~p connected", [name()]), Action = {state_timeout, 0, connected}, {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; error -> @@ -318,10 +309,6 @@ connecting(state_timeout, connected, State) -> {next_state, connected, State}; connecting(state_timeout, reconnect, _State) -> repeat_state_and_data; -connecting({call, From}, status, _State) -> - {keep_state_and_data, [{reply, From, <<"Stopped">>}]}; -connecting({call, From}, _Call, _State) -> - {keep_state_and_data, [{reply, From, <<"starting bridge ......">>}]}; connecting(info, {batch_ack, Ref}, State) -> case do_ack(State, Ref) of {ok, NewState} -> @@ -329,6 +316,10 @@ connecting(info, {batch_ack, Ref}, State) -> _ -> keep_state_and_data end; +connecting(internal, maybe_send, _State) -> + keep_state_and_data; +connecting(info, {disconnected, _Ref, _Reason}, _State) -> + keep_state_and_data; connecting(Type, Content, State) -> common(connecting, Type, Content, State). @@ -353,23 +344,23 @@ connected(internal, maybe_send, State) -> {error, NewState} -> {next_state, connecting, disconnect(NewState)} end; -connected({call, From}, ensure_started, _State) -> - {keep_state_and_data, [{reply, From, <<"bridge already started">>}]}; -connected({call, From}, status, _State) -> - {keep_state_and_data, [{reply, From, <<"Running">>}]}; connected(info, {disconnected, ConnRef, Reason}, - #{conn_ref := ConnRef, connection := Conn} = State) -> - ?INFO("Portal ~p diconnected~nreason=~p", - [name(), Conn, Reason]), - {next_state, connecting, - State#{conn_ref := undefined, - connection := undefined}}; + #{conn_ref := ConnRefCurrent, connection := Conn} = State) -> + case ConnRefCurrent =:= ConnRef of + true -> + ?INFO("Portal ~p diconnected~nreason=~p", [name(), Conn, Reason]), + {next_state, connecting, + State#{conn_ref := undefined, connection := undefined}}; + false -> + keep_state_and_data + end; connected(info, {batch_ack, Ref}, State) -> case do_ack(State, Ref) of stale -> keep_state_and_data; bad_order -> %% try re-connect then re-send + ?ERROR("Bad order ack received by portal ~p", [name()]), {next_state, connecting, disconnect(State)}; {ok, NewState} -> {keep_state, NewState, ?maybe_send} @@ -378,6 +369,8 @@ connected(Type, Content, State) -> common(connected, Type, Content, State). %% Common handlers +common(StateName, {call, From}, status, _State) -> + {keep_state_and_data, [{reply, From, StateName}]}; common(_StateName, {call, From}, ensure_started, _State) -> {keep_state_and_data, [{reply, From, ok}]}; common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> @@ -392,13 +385,13 @@ common(_StateName, {call, From}, {ensure_absent, What, Topic}, State) -> {keep_state, NewState, [{reply, From, Result}]}; common(_StateName, {call, From}, ensure_stopped, _State) -> {stop_and_reply, {shutdown, manual}, - [{reply, From, <<"stop bridge successfully">>}]}; + [{reply, From, ok}]}; common(_StateName, info, {dispatch, _, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, collect([Msg])), {keep_state, State#{replayq => NewQ}, ?maybe_send}; common(StateName, Type, Content, State) -> - ?INFO("Portal ~p discarded ~p type event at state ~p:~p", + ?INFO("Portal ~p discarded ~p type event at state ~p:\n~p", [name(), Type, StateName, Content]), {keep_state, State}. @@ -497,15 +490,16 @@ do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) -> %% this is a list of inflight BATCHes, not expecting it to be too long NewInflight = Inflight ++ [#{q_ack_ref => QAckRef, send_ack_ref => Ref, - batch => Batch - }], + batch => Batch}], {ok, State#{inflight := NewInflight}}; {error, Reason} -> ?INFO("Batch produce failed\n~p", [Reason]), {error, State} end. -do_ack(State = #{inflight := [#{send_ack_ref := Ref} | Rest]}, Ref) -> +do_ack(State = #{inflight := [#{send_ack_ref := Refx, q_ack_ref := QAckRef} | Rest], + replayq := Q}, Ref) when Refx =:= Ref -> + ok = replayq:ack(Q, QAckRef), {ok, State#{inflight := Rest}}; do_ack(#{inflight := Inflight}, Ref) -> case lists:any(fun(#{send_ack_ref := Ref0}) -> Ref0 =:= Ref end, Inflight) of @@ -533,8 +527,7 @@ disconnect(#{connection := Conn, } = State) when Conn =/= undefined -> ok = Module:stop(ConnRef, Conn), State#{conn_ref => undefined, - connection => undefined - }; + connection => undefined}; disconnect(State) -> State. %% Called only when replayq needs to dump it to disk. diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index f47a0cf5a..d466a970d 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -74,8 +74,8 @@ start(Config = #{address := Address}) -> end. stop(Ref, #{ack_collector := AckCollector, client_pid := Pid}) -> - safe_stop(AckCollector, fun() -> AckCollector ! ?STOP(Ref) end, 1000), safe_stop(Pid, fun() -> emqx_client:stop(Pid) end, 1000), + safe_stop(AckCollector, fun() -> AckCollector ! ?STOP(Ref) end, 1000), ok. ensure_subscribed(#{client_pid := Pid}, Topic, QoS) when is_pid(Pid) -> @@ -120,7 +120,7 @@ send(#{client_pid := ClientPid, ack_collector := AckCollector} = Conn, [Msg | Re {ok, PktId} -> send(Conn, Rest, [PktId | Acc]); {error, {_PacketId, inflight_full}} -> - timer:sleep(100), + timer:sleep(10), send(Conn, Batch, Acc); {error, Reason} -> %% NOTE: There is no partial sucess of a batch and recover from the middle @@ -176,7 +176,7 @@ import_msg(Msg) -> make_hdlr(Parent, AckCollector, Ref) -> #{puback => fun(Ack) -> handle_puback(AckCollector, Ack) end, publish => fun(Msg) -> import_msg(Msg) end, - disconnected => fun(RC, _Properties) -> Parent ! {disconnected, Ref, RC}, ok end + disconnected => fun(Reason) -> Parent ! {disconnected, Ref, Reason}, ok end }. subscribe_remote_topics(ClientPid, Subscriptions) -> diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl index fbc292c4e..845136b7a 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/portal/emqx_portal_sup.erl @@ -16,7 +16,7 @@ -behavior(supervisor). -export([start_link/0, start_link/1, portals/0]). - +-export([create_portal/2, drop_portal/1]). -export([init/1]). -define(SUP, ?MODULE). @@ -46,3 +46,15 @@ portal_spec({Name, Config}) -> -spec(portals() -> [{node(), map()}]). portals() -> [{Name, emqx_portal:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. + +create_portal(Id, Config) -> + supervisor:start_child(?SUP, portal_spec({Id, Config})). + +drop_portal(Id) -> + case supervisor:terminate_child(?SUP, Id) of + ok -> + supervisor:delete_child(?SUP, Id); + Error -> + emqx_logger:error("[Bridge] Delete bridge failed", [Error]), + Error + end. diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index 3d34eda50..d9e7b38a2 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -146,8 +146,6 @@ t_mqtt(Config) when is_list(Config) -> ?assertEqual([{ForwardedTopic, 1}], emqx_portal:get_subscriptions(Pid)), ok = emqx_portal:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), - %% TODO: investigate why it's necessary - timer:sleep(1000), ?assertEqual([{ForwardedTopic, 1}, {ForwardedTopic2, 1}], emqx_portal:get_subscriptions(Pid)), {ok, ConnPid} = emqx_mock_client:start_link(ClientId), @@ -182,13 +180,16 @@ receive_and_match_messages(Ref, Msgs) -> ok. do_receive_and_match_messages(_Ref, []) -> ok; -do_receive_and_match_messages(Ref, [I | Rest]) -> +do_receive_and_match_messages(Ref, [I | Rest] = Exp) -> receive {Ref, timeout} -> erlang:error(timeout); {Ref, [#{payload := P} = Msg]} -> - case I =:= binary_to_integer(P) of - true -> ok; - false -> throw({unexpected, Msg, [I | Rest]}) - end, - do_receive_and_match_messages(Ref, Rest) + case binary_to_integer(P) of + I -> %% exact match + do_receive_and_match_messages(Ref, Rest); + J when J < I -> %% allow retry + do_receive_and_match_messages(Ref, Exp); + _Other -> + throw({unexpected, Msg, Exp}) + end end. diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index e9a3583fe..18d7ed4cd 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -139,7 +139,8 @@ match_nums([#message{payload = P} | Rest], Nums) -> I = binary_to_integer(P), case Nums of [I | NumsLeft] -> match_nums(Rest, NumsLeft); - _ -> error({I, Nums}) + [J | _] when J > I -> match_nums(Rest, Nums); %% allow retry + _ -> error([{received, I}, {expecting, Nums}]) end. make_config(Ref, TestPid, Result) -> From 911a8138917ad7805c7a4feb511738128e18ffc0 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Wed, 27 Feb 2019 10:46:30 +0800 Subject: [PATCH 23/26] Fix copyright and unify log method --- include/emqx_client.hrl | 2 +- include/logger.hrl | 2 ++ src/emqx_logger_formatter.erl | 2 +- src/emqx_portal_connect.erl | 6 +++--- src/emqx_rpc.erl | 3 +-- src/emqx_types.erl | 3 +-- src/portal/emqx_portal.erl | 14 +++++++------- src/portal/emqx_portal_mqtt.erl | 2 +- src/portal/emqx_portal_msg.erl | 3 +-- src/portal/emqx_portal_rpc.erl | 3 +-- src/portal/emqx_portal_sup.erl | 6 ++++-- test/emqx_mqtt_packet_SUITE.erl | 4 ++-- test/emqx_portal_SUITE.erl | 2 +- test/emqx_portal_mqtt_tests.erl | 2 +- test/emqx_portal_rpc_tests.erl | 2 +- test/emqx_portal_tests.erl | 2 +- test/emqx_protocol_SUITE.erl | 2 +- 17 files changed, 30 insertions(+), 30 deletions(-) diff --git a/include/emqx_client.hrl b/include/emqx_client.hrl index ce66c98d0..535b8ad55 100644 --- a/include/emqx_client.hrl +++ b/include/emqx_client.hrl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/include/logger.hrl b/include/logger.hrl index f87668cc7..bfa405170 100644 --- a/include/logger.hrl +++ b/include/logger.hrl @@ -39,3 +39,5 @@ begin (logger:log(Level,#{},#{report_cb => fun(_) -> {(Format), (Args)} end})) end). + +-define(LOG(Level, Format), ?LOG(Level, Format, [])). diff --git a/src/emqx_logger_formatter.erl b/src/emqx_logger_formatter.erl index 034eb8d36..dd94cceb6 100644 --- a/src/emqx_logger_formatter.erl +++ b/src/emqx_logger_formatter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017-2013-2019. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_portal_connect.erl b/src/emqx_portal_connect.erl index f0b7474a5..9c2d168f3 100644 --- a/src/emqx_portal_connect.erl +++ b/src/emqx_portal_connect.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -54,8 +54,8 @@ start(Module, Config) -> {ok, Ref, Conn}; {error, Reason} -> Config1 = obfuscate(Config), - ?ERROR("Failed to connect with module=~p\n" - "config=~p\nreason:~p", [Module, Config1, Reason]), + ?LOG(error, "Failed to connect with module=~p\n" + "config=~p\nreason:~p", [Module, Config1, Reason]), error end. diff --git a/src/emqx_rpc.erl b/src/emqx_rpc.erl index 0245da838..e0d82f400 100644 --- a/src/emqx_rpc.erl +++ b/src/emqx_rpc.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,4 +28,3 @@ multicall(Nodes, Mod, Fun, Args) -> cast(Node, Mod, Fun, Args) -> ?RPC:cast(Node, Mod, Fun, Args). - diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 904e1df91..2fe34e853 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -31,7 +31,7 @@ -type(pubsub() :: publish | subscribe). -type(topic() :: binary()). -type(subid() :: binary() | atom()). --type(subopts() :: #{qos := eqmx_mqtt_types:qos(), +-type(subopts() :: #{qos := emqx_mqtt_types:qos(), share => binary(), atom() => term() }). @@ -59,4 +59,3 @@ -type(alarm() :: #alarm{}). -type(plugin() :: #plugin{}). -type(command() :: #command{}). - diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 83a29521d..8ef764c39 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -279,7 +279,7 @@ standing_by(state_timeout, do_connect, State) -> standing_by({call, From}, _Call, _State) -> {keep_state_and_data, [{reply, From, {error,standing_by}}]}; standing_by(info, Info, State) -> - ?INFO("Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), + ?LOG(info, "Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), {keep_state_and_data, State}; standing_by(Type, Content, State) -> common(standing_by, Type, Content, State). @@ -298,7 +298,7 @@ connecting(enter, _, #{reconnect_delay_ms := Timeout, ok = subscribe_local_topics(Forwards), case ConnectFun(Subs) of {ok, ConnRef, Conn} -> - ?INFO("Portal ~p connected", [name()]), + ?LOG(info, "Portal ~p connected", [name()]), Action = {state_timeout, 0, connected}, {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; error -> @@ -348,7 +348,7 @@ connected(info, {disconnected, ConnRef, Reason}, #{conn_ref := ConnRefCurrent, connection := Conn} = State) -> case ConnRefCurrent =:= ConnRef of true -> - ?INFO("Portal ~p diconnected~nreason=~p", [name(), Conn, Reason]), + ?LOG(info, "Portal ~p diconnected~nreason=~p", [name(), Conn, Reason]), {next_state, connecting, State#{conn_ref := undefined, connection := undefined}}; false -> @@ -360,7 +360,7 @@ connected(info, {batch_ack, Ref}, State) -> keep_state_and_data; bad_order -> %% try re-connect then re-send - ?ERROR("Bad order ack received by portal ~p", [name()]), + ?LOG(error, "Bad order ack received by portal ~p", [name()]), {next_state, connecting, disconnect(State)}; {ok, NewState} -> {keep_state, NewState, ?maybe_send} @@ -391,7 +391,7 @@ common(_StateName, info, {dispatch, _, Msg}, NewQ = replayq:append(Q, collect([Msg])), {keep_state, State#{replayq => NewQ}, ?maybe_send}; common(StateName, Type, Content, State) -> - ?INFO("Portal ~p discarded ~p type event at state ~p:\n~p", + ?LOG(info, "Portal ~p discarded ~p type event at state ~p:\n~p", [name(), Type, StateName, Content]), {keep_state, State}. @@ -462,7 +462,7 @@ retry_inflight(#{inflight := Inflight} = State, {ok, NewState} -> retry_inflight(NewState, T); {error, Reason} -> - ?ERROR("Inflight retry failed\n~p", [Reason]), + ?LOG(error, "Inflight retry failed\n~p", [Reason]), {error, State#{inflight := Inflight ++ Remain}} end. @@ -493,7 +493,7 @@ do_send(State = #{inflight := Inflight}, QAckRef, [_ | _] = Batch) -> batch => Batch}], {ok, State#{inflight := NewInflight}}; {error, Reason} -> - ?INFO("Batch produce failed\n~p", [Reason]), + ?LOG(info, "Batch produce failed\n~p", [Reason]), {error, State} end. diff --git a/src/portal/emqx_portal_mqtt.erl b/src/portal/emqx_portal_mqtt.erl index d466a970d..3d0b90e30 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/portal/emqx_portal_mqtt.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/portal/emqx_portal_msg.erl b/src/portal/emqx_portal_msg.erl index f8554f0b6..45cd64154 100644 --- a/src/portal/emqx_portal_msg.erl +++ b/src/portal/emqx_portal_msg.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -82,4 +82,3 @@ to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, emqx_message:make(portal, QoS, Topic, Payload))). topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). - diff --git a/src/portal/emqx_portal_rpc.erl b/src/portal/emqx_portal_rpc.erl index fcd8b24e9..8b5600f77 100644 --- a/src/portal/emqx_portal_rpc.erl +++ b/src/portal/emqx_portal_rpc.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -103,4 +103,3 @@ poke(Node) -> Node -> ok; {badrpc, Reason} -> {error, Reason} end. - diff --git a/src/portal/emqx_portal_sup.erl b/src/portal/emqx_portal_sup.erl index 845136b7a..271c60423 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/portal/emqx_portal_sup.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ -module(emqx_portal_sup). -behavior(supervisor). +-include("logger.hrl"). + -export([start_link/0, start_link/1, portals/0]). -export([create_portal/2, drop_portal/1]). -export([init/1]). @@ -55,6 +57,6 @@ drop_portal(Id) -> ok -> supervisor:delete_child(?SUP, Id); Error -> - emqx_logger:error("[Bridge] Delete bridge failed", [Error]), + ?LOG(error, "[Bridge] Delete bridge failed", [Error]), Error end. diff --git a/test/emqx_mqtt_packet_SUITE.erl b/test/emqx_mqtt_packet_SUITE.erl index 3be2617b0..0cd544c25 100644 --- a/test/emqx_mqtt_packet_SUITE.erl +++ b/test/emqx_mqtt_packet_SUITE.erl @@ -1,5 +1,5 @@ %%%=================================================================== -%%% Copyright (c) 2013-2013-2019 EMQ Inc. All rights reserved. +%%% Copyright (c) 2013-2019 EMQ Inc. All rights reserved. %%% %%% Licensed under the Apache License, Version 2.0 (the "License"); %%% you may not use this file except in compliance with the License. @@ -90,7 +90,7 @@ case1_protocol_name(_) -> {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data), Disconnect = gen_tcp:recv(Sock, 0), ?assertEqual({error, closed}, Disconnect). - + case2_protocol_ver(_) -> {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), Packet = serialize(?CASE2_PROTOCAL_VER), diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_portal_SUITE.erl index d9e7b38a2..2ceb7ea1e 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_portal_SUITE.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_portal_mqtt_tests.erl index 788f5a4b2..1fcd71e79 100644 --- a/test/emqx_portal_mqtt_tests.erl +++ b/test/emqx_portal_mqtt_tests.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_portal_rpc_tests.erl b/test/emqx_portal_rpc_tests.erl index 5fd7608c0..363a0ee5a 100644 --- a/test/emqx_portal_rpc_tests.erl +++ b/test/emqx_portal_rpc_tests.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_portal_tests.erl b/test/emqx_portal_tests.erl index 18d7ed4cd..7cca66adc 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_portal_tests.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_protocol_SUITE.erl b/test/emqx_protocol_SUITE.erl index e48825d76..1ebb90876 100644 --- a/test/emqx_protocol_SUITE.erl +++ b/test/emqx_protocol_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2013-2013-2019 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2013-2019 EMQ Enterprise, Inc. (http://emqtt.io) %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. From d18f4ba55056a922ac3988ca7396c66d0a40ef1d Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Thu, 28 Feb 2019 11:08:13 +0800 Subject: [PATCH 24/26] Fix wrong config entries --- etc/emqx.conf | 21 +++++++++++++-------- priv/emqx.schema | 8 ++++++-- src/portal/emqx_portal.erl | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 8dc510a31..2668c41a2 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1723,16 +1723,21 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Inflight size. ## ## Value: Integer -## bridge.aws.max_inflight = 32 +## bridge.aws.max_inflight_batches = 32 -## Maximum number of messages in one batch when sending to remote borkers -## NOTE: when bridging via MQTT connection to remote broker, this config is only -## used for internal message passing optimization as the underlying MQTT -## protocol does not supports batching. +## Max number of messages to collect in a batch for +## each send call towards emqx_portal_connect ## ## Value: Integer ## default: 32 -## bridge.aws.queue.batch_size = 32 +## bridge.aws.queue.batch_count_limit = 32 + +## Max number of bytes to collect in a batch for each +## send call towards emqx_portal_connect +## +## Value: Bytesize +## default: 1000M +## bridge.aws.queue.batch_bytes_limit = 1000MB ## Base directory for replayq to store messages on disk ## If this config entry is missing or set to undefined, @@ -1767,7 +1772,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## The ClientId of a remote bridge. ## ## Value: String -## bridge.azure.client_id = bridge_aws +## bridge.azure.client_id = bridge_azure ## The Clean start flag of a remote bridge. ## @@ -1876,7 +1881,7 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## Inflight size. ## ## Value: Integer -## bridge.azure.max_inflight = 32 +## bridge.azure.max_inflight_batches = 32 ## Maximum number of messages in one batch when sending to remote borkers ## NOTE: when bridging via MQTT connection to remote broker, this config is only diff --git a/priv/emqx.schema b/priv/emqx.schema index 388b9beea..4ca43f232 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1600,15 +1600,19 @@ end}. {datatype, {duration, ms}} ]}. -{mapping, "bridge.$name.max_inflight", "emqx.bridges", [ +{mapping, "bridge.$name.max_inflight_batches", "emqx.bridges", [ {default, 0}, {datatype, integer} ]}. -{mapping, "bridge.$name.queue.batch_size", "emqx.bridges", [ +{mapping, "bridge.$name.queue.batch_count_limit", "emqx.bridges", [ {datatype, integer} ]}. +{mapping, "bridge.$name.queue.batch_bytes_limit", "emqx.bridges", [ + {datatype, bytesize} +]}. + {mapping, "bridge.$name.queue.replayq_dir", "emqx.bridges", [ {datatype, string} ]}. diff --git a/src/portal/emqx_portal.erl b/src/portal/emqx_portal.erl index 8ef764c39..4a6ad2437 100644 --- a/src/portal/emqx_portal.erl +++ b/src/portal/emqx_portal.erl @@ -114,9 +114,9 @@ %% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each %% send call towards emqx_portal_connect %% queue.batch_count_limit: Max number of messages to collect in a batch for -%% each send call towards eqmx_portal_connect +%% each send call towards emqx_portal_connect %% queue.replayq_dir: Directory where replayq should persist messages -%% queue.replayq_seg_bytes: Size in bytes for each replqyq segnment file +%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file %% %% Find more connection specific configs in the callback modules %% of emqx_portal_connect behaviour. From 7efd7b3ec05382bdc55c8de19151f20af0cf3231 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Thu, 28 Feb 2019 15:31:54 +0800 Subject: [PATCH 25/26] Rename portal to bridge --- Makefile | 20 +++--- etc/emqx.conf | 4 +- priv/emqx.schema | 4 +- .../emqx_bridge.erl} | 62 +++++++++---------- .../emqx_bridge_mqtt.erl} | 10 +-- .../emqx_bridge_msg.erl} | 8 +-- .../emqx_bridge_rpc.erl} | 16 ++--- .../emqx_bridge_sup.erl} | 28 ++++----- ...al_connect.erl => emqx_bridge_connect.erl} | 6 +- src/emqx_sup.erl | 5 +- ...portal_SUITE.erl => emqx_bridge_SUITE.erl} | 60 +++++++++--------- ...t_tests.erl => emqx_bridge_mqtt_tests.erl} | 8 +-- ...pc_tests.erl => emqx_bridge_rpc_tests.erl} | 14 ++--- ...portal_tests.erl => emqx_bridge_tests.erl} | 56 ++++++++--------- test/emqx_ct_broker_helpers.erl | 6 +- 15 files changed, 153 insertions(+), 154 deletions(-) rename src/{portal/emqx_portal.erl => bridge/emqx_bridge.erl} (91%) rename src/{portal/emqx_portal_mqtt.erl => bridge/emqx_bridge_mqtt.erl} (96%) rename src/{portal/emqx_portal_msg.erl => bridge/emqx_bridge_msg.erl} (94%) rename src/{portal/emqx_portal_rpc.erl => bridge/emqx_bridge_rpc.erl} (88%) rename src/{portal/emqx_portal_sup.erl => bridge/emqx_bridge_sup.erl} (70%) rename src/{emqx_portal_connect.erl => emqx_bridge_connect.erl} (94%) rename test/{emqx_portal_SUITE.erl => emqx_bridge_SUITE.erl} (79%) rename test/{emqx_portal_mqtt_tests.erl => emqx_bridge_mqtt_tests.erl} (91%) rename test/{emqx_portal_rpc_tests.erl => emqx_bridge_rpc_tests.erl} (80%) rename test/{emqx_portal_tests.erl => emqx_bridge_tests.erl} (75%) diff --git a/Makefile b/Makefile index 9aa90f94c..35c2c0bee 100644 --- a/Makefile +++ b/Makefile @@ -27,17 +27,17 @@ TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx EUNIT_OPTS = verbose -# CT_SUITES = emqx_frame +CT_SUITES = emqx_bridge ## emqx_trie emqx_router emqx_frame emqx_mqtt_compat -CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ - emqx_access emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \ - emqx_keepalive emqx_lib emqx_metrics emqx_mod emqx_mod_sup emqx_mqtt_caps \ - emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ - emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ - emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_portal \ - emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ - emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message +# CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ +# emqx_access emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \ +# emqx_keepalive emqx_lib emqx_metrics emqx_mod emqx_mod_sup emqx_mqtt_caps \ +# emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ +# emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ +# emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ +# emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc emqx_ws_connection \ +# emqx_packet emqx_connection emqx_tracer emqx_sys_mon emqx_message CT_NODE_NAME = emqxct@127.0.0.1 CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) @@ -110,7 +110,7 @@ rebar-ct: rebar-ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') ## Run one single CT with rebar3 -## e.g. make ct-one-suite suite=emqx_portal +## e.g. make ct-one-suite suite=emqx_bridge ct-one-suite: rebar-ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE diff --git a/etc/emqx.conf b/etc/emqx.conf index 2668c41a2..92d4f59b8 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1726,14 +1726,14 @@ listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-G ## bridge.aws.max_inflight_batches = 32 ## Max number of messages to collect in a batch for -## each send call towards emqx_portal_connect +## each send call towards emqx_bridge_connect ## ## Value: Integer ## default: 32 ## bridge.aws.queue.batch_count_limit = 32 ## Max number of bytes to collect in a batch for each -## send call towards emqx_portal_connect +## send call towards emqx_bridge_connect ## ## Value: Bytesize ## default: 1000M diff --git a/priv/emqx.schema b/priv/emqx.schema index 4ca43f232..e490e024f 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1675,9 +1675,9 @@ end}. true when Subs =/= [] -> error({"subscriptions are not supported when bridging between emqx nodes", Name, Subs}); true -> - emqx_portal_rpc; + emqx_bridge_rpc; false -> - emqx_portal_mqtt + emqx_bridge_mqtt end end, %% to be backward compatible diff --git a/src/portal/emqx_portal.erl b/src/bridge/emqx_bridge.erl similarity index 91% rename from src/portal/emqx_portal.erl rename to src/bridge/emqx_bridge.erl index 4a6ad2437..af85df68e 100644 --- a/src/portal/emqx_portal.erl +++ b/src/bridge/emqx_bridge.erl @@ -12,18 +12,18 @@ %% See the License for the specific language governing permissions and %% limitations under the License. -%% @doc Portal works in two layers (1) batching layer (2) transport layer -%% The `portal' batching layer collects local messages in batches and sends over +%% @doc Bridge works in two layers (1) batching layer (2) transport layer +%% The `bridge' batching layer collects local messages in batches and sends over %% to remote MQTT node/cluster via `connetion' transport layer. %% In case `REMOTE' is also an EMQX node, `connection' is recommended to be -%% the `gen_rpc' based implementation `emqx_portal_rpc'. Otherwise `connection' -%% has to be `emqx_portal_mqtt'. +%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection' +%% has to be `emqx_bridge_mqtt'. %% %% ``` %% +------+ +--------+ %% | EMQX | | REMOTE | %% | | | | -%% | (portal) <==(connection)==> | | +%% | (bridge) <==(connection)==> | | %% | | | | %% | | | | %% +------+ +--------+ @@ -47,8 +47,8 @@ %% (3): received {disconnected, conn_ref(), Reason} OR %% failed to send to remote node/cluster. %% -%% NOTE: A portal worker may subscribe to multiple (including wildcard) -%% local topics, and the underlying `emqx_portal_connect' may subscribe to +%% NOTE: A bridge worker may subscribe to multiple (including wildcard) +%% local topics, and the underlying `emqx_bridge_connect' may subscribe to %% multiple remote topics, however, worker/connections are not designed %% to support automatic load-balancing, i.e. in case it can not keep up %% with the amount of messages comming in, administrator should split and @@ -57,7 +57,7 @@ %% NOTES: %% * Local messages are all normalised to QoS-1 when exporting to remote --module(emqx_portal). +-module(emqx_bridge). -behaviour(gen_statem). %% APIs @@ -84,7 +84,7 @@ -type id() :: atom() | string() | pid(). -type qos() :: emqx_mqtt_types:qos(). -type config() :: map(). --type batch() :: [emqx_portal_msg:exp_msg()]. +-type batch() :: [emqx_bridge_msg:exp_msg()]. -type ack_ref() :: term(). -type topic() :: emqx_topic:topic(). @@ -99,12 +99,12 @@ -define(DEFAULT_SEG_BYTES, (1 bsl 20)). -define(maybe_send, {next_event, internal, maybe_send}). -%% @doc Start a portal worker. Supported configs: -%% start_type: 'manual' (default) or 'auto', when manual, portal will stay +%% @doc Start a bridge worker. Supported configs: +%% start_type: 'manual' (default) or 'auto', when manual, bridge will stay %% at 'standing_by' state until a manual call to start it. -%% connect_module: The module which implements emqx_portal_connect behaviour +%% connect_module: The module which implements emqx_bridge_connect behaviour %% and work as message batch transport layer -%% reconnect_delay_ms: Delay in milli-seconds for the portal worker to retry +%% reconnect_delay_ms: Delay in milli-seconds for the bridge worker to retry %% in case of transportation failure. %% max_inflight_batches: Max number of batches allowed to send-ahead before %% receiving confirmation from remote node/cluster @@ -112,20 +112,20 @@ %% `undefined', `<<>>' or `""' to disable %% forwards: Local topics to subscribe. %% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each -%% send call towards emqx_portal_connect +%% send call towards emqx_bridge_connect %% queue.batch_count_limit: Max number of messages to collect in a batch for -%% each send call towards emqx_portal_connect +%% each send call towards emqx_bridge_connect %% queue.replayq_dir: Directory where replayq should persist messages %% queue.replayq_seg_bytes: Size in bytes for each replayq segment file %% %% Find more connection specific configs in the callback modules -%% of emqx_portal_connect behaviour. +%% of emqx_bridge_connect behaviour. start_link(Name, Config) when is_list(Config) -> start_link(Name, maps:from_list(Config)); start_link(Name, Config) -> gen_statem:start_link({local, name(Name)}, ?MODULE, Config, []). -%% @doc Manually start portal worker. State idempotency ensured. +%% @doc Manually start bridge worker. State idempotency ensured. ensure_started(Name) -> gen_statem:call(name(Name), ensure_started). @@ -135,7 +135,7 @@ ensure_started(Name, Config) -> {error, {already_started,Pid}} -> {ok, Pid} end. -%% @doc Manually stop portal worker. State idempotency ensured. +%% @doc Manually stop bridge worker. State idempotency ensured. ensure_stopped(Id) -> ensure_stopped(Id, 1000). @@ -168,7 +168,7 @@ status(Pid) -> %% @doc This function is to be evaluated on message/batch receiver side. -spec import_batch(batch(), fun(() -> ok)) -> ok. import_batch(Batch, AckFun) -> - lists:foreach(fun emqx_broker:publish/1, emqx_portal_msg:to_broker_msgs(Batch)), + lists:foreach(fun emqx_broker:publish/1, emqx_bridge_msg:to_broker_msgs(Batch)), AckFun(). %% @doc This function is to be evaluated on message/batch exporter side @@ -197,14 +197,14 @@ ensure_forward_absent(Id, Topic) -> gen_statem:call(id(Id), {ensure_absent, forwards, topic(Topic)}). %% @doc Ensure subscribed to remote topic. -%% NOTE: only applicable when connection module is emqx_portal_mqtt +%% NOTE: only applicable when connection module is emqx_bridge_mqtt %% return `{error, no_remote_subscription_support}' otherwise. -spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. ensure_subscription_present(Id, Topic, QoS) -> gen_statem:call(id(Id), {ensure_present, subscriptions, {topic(Topic), QoS}}). %% @doc Ensure unsubscribed from remote topic. -%% NOTE: only applicable when connection module is emqx_portal_mqtt +%% NOTE: only applicable when connection module is emqx_bridge_mqtt -spec ensure_subscription_absent(id(), topic()) -> ok. ensure_subscription_absent(Id, Topic) -> gen_statem:call(id(Id), {ensure_absent, subscriptions, topic(Topic)}). @@ -225,7 +225,7 @@ init(Config) -> seg_bytes => GetQ(replayq_seg_bytes, ?DEFAULT_SEG_BYTES) } end, - Queue = replayq:open(QueueConfig#{sizer => fun emqx_portal_msg:estimate_size/1, + Queue = replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1, marshaller => fun msg_marshaller/1}), Topics = lists:sort([iolist_to_binary(T) || T <- Get(forwards, [])]), Subs = lists:keysort(1, lists:map(fun({T0, QoS}) -> @@ -241,7 +241,7 @@ init(Config) -> mountpoint, forwards ], Config#{subscriptions => Subs}), - ConnectFun = fun(SubsX) -> emqx_portal_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end, + ConnectFun = fun(SubsX) -> emqx_bridge_connect:start(ConnectModule, ConnectConfig#{subscriptions := SubsX}) end, {ok, standing_by, #{connect_module => ConnectModule, connect_fun => ConnectFun, @@ -279,7 +279,7 @@ standing_by(state_timeout, do_connect, State) -> standing_by({call, From}, _Call, _State) -> {keep_state_and_data, [{reply, From, {error,standing_by}}]}; standing_by(info, Info, State) -> - ?LOG(info, "Portal ~p discarded info event at state standing_by:\n~p", [name(), Info]), + ?LOG(info, "Bridge ~p discarded info event at state standing_by:\n~p", [name(), Info]), {keep_state_and_data, State}; standing_by(Type, Content, State) -> common(standing_by, Type, Content, State). @@ -298,7 +298,7 @@ connecting(enter, _, #{reconnect_delay_ms := Timeout, ok = subscribe_local_topics(Forwards), case ConnectFun(Subs) of {ok, ConnRef, Conn} -> - ?LOG(info, "Portal ~p connected", [name()]), + ?LOG(info, "Bridge ~p connected", [name()]), Action = {state_timeout, 0, connected}, {keep_state, State#{conn_ref => ConnRef, connection => Conn}, Action}; error -> @@ -348,7 +348,7 @@ connected(info, {disconnected, ConnRef, Reason}, #{conn_ref := ConnRefCurrent, connection := Conn} = State) -> case ConnRefCurrent =:= ConnRef of true -> - ?LOG(info, "Portal ~p diconnected~nreason=~p", [name(), Conn, Reason]), + ?LOG(info, "Bridge ~p diconnected~nreason=~p", [name(), Conn, Reason]), {next_state, connecting, State#{conn_ref := undefined, connection := undefined}}; false -> @@ -360,7 +360,7 @@ connected(info, {batch_ack, Ref}, State) -> keep_state_and_data; bad_order -> %% try re-connect then re-send - ?LOG(error, "Bad order ack received by portal ~p", [name()]), + ?LOG(error, "Bad order ack received by bridge ~p", [name()]), {next_state, connecting, disconnect(State)}; {ok, NewState} -> {keep_state, NewState, ?maybe_send} @@ -391,7 +391,7 @@ common(_StateName, info, {dispatch, _, Msg}, NewQ = replayq:append(Q, collect([Msg])), {keep_state, State#{replayq => NewQ}, ?maybe_send}; common(StateName, Type, Content, State) -> - ?LOG(info, "Portal ~p discarded ~p type event at state ~p:\n~p", + ?LOG(info, "Bridge ~p discarded ~p type event at state ~p:\n~p", [name(), Type, StateName, Content]), {keep_state, State}. @@ -531,15 +531,15 @@ disconnect(#{connection := Conn, disconnect(State) -> State. %% Called only when replayq needs to dump it to disk. -msg_marshaller(Bin) when is_binary(Bin) -> emqx_portal_msg:from_binary(Bin); -msg_marshaller(Msg) -> emqx_portal_msg:to_binary(Msg). +msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin); +msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg). %% Return {ok, SendAckRef} or {error, Reason} maybe_send(#{connect_module := Module, connection := Connection, mountpoint := Mountpoint }, Batch) -> - Module:send(Connection, [emqx_portal_msg:to_export(Module, Mountpoint, M) || M <- Batch]). + Module:send(Connection, [emqx_bridge_msg:to_export(Module, Mountpoint, M) || M <- Batch]). format_mountpoint(undefined) -> undefined; diff --git a/src/portal/emqx_portal_mqtt.erl b/src/bridge/emqx_bridge_mqtt.erl similarity index 96% rename from src/portal/emqx_portal_mqtt.erl rename to src/bridge/emqx_bridge_mqtt.erl index 3d0b90e30..590fbabb7 100644 --- a/src/portal/emqx_portal_mqtt.erl +++ b/src/bridge/emqx_bridge_mqtt.erl @@ -12,10 +12,10 @@ %% See the License for the specific language governing permissions and %% limitations under the License. -%% @doc This module implements EMQX Portal transport layer on top of MQTT protocol +%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol --module(emqx_portal_mqtt). --behaviour(emqx_portal_connect). +-module(emqx_bridge_mqtt). +-behaviour(emqx_bridge_connect). %% behaviour callbacks -export([start/1, @@ -154,7 +154,7 @@ match_acks(Parent, Acked, Sent) -> match_acks_1(_Parent, {empty, Empty}, Sent) -> {Empty, Sent}; match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId]) | Sent]) -> %% batch finished - ok = emqx_portal:handle_ack(Parent, Ref), + ok = emqx_bridge:handle_ack(Parent, Ref), match_acks(Parent, Acked, Sent); match_acks_1(Parent, {{value, PktId}, Acked}, [?REF_IDS(Ref, [PktId | RestIds]) | Sent]) -> %% one message finished, but not the whole batch @@ -171,7 +171,7 @@ handle_puback(AckCollector, #{packet_id := PktId, reason_code := RC}) -> %% Message published from remote broker. Import to local broker. import_msg(Msg) -> %% auto-ack should be enabled in emqx_client, hence dummy ack-fun. - emqx_portal:import_batch([Msg], _AckFun = fun() -> ok end). + emqx_bridge:import_batch([Msg], _AckFun = fun() -> ok end). make_hdlr(Parent, AckCollector, Ref) -> #{puback => fun(Ack) -> handle_puback(AckCollector, Ack) end, diff --git a/src/portal/emqx_portal_msg.erl b/src/bridge/emqx_bridge_msg.erl similarity index 94% rename from src/portal/emqx_portal_msg.erl rename to src/bridge/emqx_bridge_msg.erl index 45cd64154..6633027f9 100644 --- a/src/portal/emqx_portal_msg.erl +++ b/src/bridge/emqx_bridge_msg.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_msg). +-module(emqx_bridge_msg). -export([ to_binary/1 , from_binary/1 @@ -37,9 +37,9 @@ %% Shame that we have to know the callback module here %% would be great if we can get rid of #mqtt_msg{} record %% and use #message{} in all places. --spec to_export(emqx_portal_rpc | emqx_portal_mqtt, +-spec to_export(emqx_bridge_rpc | emqx_bridge_mqtt, undefined | binary(), msg()) -> exp_msg(). -to_export(emqx_portal_mqtt, Mountpoint, +to_export(emqx_bridge_mqtt, Mountpoint, #message{topic = Topic, payload = Payload, flags = Flags @@ -79,6 +79,6 @@ to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, %% published from remote node over a MQTT connection emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, - emqx_message:make(portal, QoS, Topic, Payload))). + emqx_message:make(bridge, QoS, Topic, Payload))). topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/src/portal/emqx_portal_rpc.erl b/src/bridge/emqx_bridge_rpc.erl similarity index 88% rename from src/portal/emqx_portal_rpc.erl rename to src/bridge/emqx_bridge_rpc.erl index 8b5600f77..b818d65da 100644 --- a/src/portal/emqx_portal_rpc.erl +++ b/src/bridge/emqx_bridge_rpc.erl @@ -12,10 +12,10 @@ %% See the License for the specific language governing permissions and %% limitations under the License. -%% @doc This module implements EMQX Portal transport layer based on gen_rpc. +%% @doc This module implements EMQX Bridge transport layer based on gen_rpc. --module(emqx_portal_rpc). --behaviour(emqx_portal_connect). +-module(emqx_bridge_rpc). +-behaviour(emqx_bridge_connect). %% behaviour callbacks -export([start/1, @@ -29,8 +29,8 @@ , heartbeat/2 ]). --type ack_ref() :: emqx_portal:ack_ref(). --type batch() :: emqx_portal:batch(). +-type ack_ref() :: emqx_bridge:ack_ref(). +-type batch() :: emqx_bridge:batch(). -define(HEARTBEAT_INTERVAL, timer:seconds(1)). @@ -58,7 +58,7 @@ stop(Pid, _Remote) when is_pid(Pid) -> end, ok. -%% @doc Callback for `emqx_portal_connect' behaviour +%% @doc Callback for `emqx_bridge_connect' behaviour -spec send(node(), batch()) -> {ok, ack_ref()} | {error, any()}. send(Remote, Batch) -> Sender = self(), @@ -73,14 +73,14 @@ handle_send(SenderPid, Batch) -> SenderNode = node(SenderPid), Ref = make_ref(), AckFun = fun() -> ?RPC:cast(SenderNode, ?MODULE, handle_ack, [SenderPid, Ref]), ok end, - case emqx_portal:import_batch(Batch, AckFun) of + case emqx_bridge:import_batch(Batch, AckFun) of ok -> {ok, Ref}; Error -> Error end. %% @doc Handle batch ack in sender node. handle_ack(SenderPid, Ref) -> - ok = emqx_portal:handle_ack(SenderPid, Ref). + ok = emqx_bridge:handle_ack(SenderPid, Ref). %% @hidden Heartbeat loop heartbeat(Parent, RemoteNode) -> diff --git a/src/portal/emqx_portal_sup.erl b/src/bridge/emqx_bridge_sup.erl similarity index 70% rename from src/portal/emqx_portal_sup.erl rename to src/bridge/emqx_bridge_sup.erl index 271c60423..bcacb411c 100644 --- a/src/portal/emqx_portal_sup.erl +++ b/src/bridge/emqx_bridge_sup.erl @@ -12,17 +12,17 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_sup). +-module(emqx_bridge_sup). -behavior(supervisor). -include("logger.hrl"). --export([start_link/0, start_link/1, portals/0]). --export([create_portal/2, drop_portal/1]). +-export([start_link/0, start_link/1, bridges/0]). +-export([create_bridge/2, drop_bridge/1]). -export([init/1]). -define(SUP, ?MODULE). --define(WORKER_SUP, emqx_portal_worker_sup). +-define(WORKER_SUP, emqx_bridge_worker_sup). start_link() -> start_link(?SUP). @@ -31,28 +31,28 @@ start_link(Name) -> init(?SUP) -> BridgesConf = emqx_config:get_env(bridges, []), - BridgeSpec = lists:map(fun portal_spec/1, BridgesConf), + BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), SupFlag = #{strategy => one_for_one, intensity => 100, period => 10}, {ok, {SupFlag, BridgeSpec}}. -portal_spec({Name, Config}) -> +bridge_spec({Name, Config}) -> #{id => Name, - start => {emqx_portal, start_link, [Name, Config]}, + start => {emqx_bridge, start_link, [Name, Config]}, restart => permanent, shutdown => 5000, type => worker, - modules => [emqx_portal]}. + modules => [emqx_bridge]}. --spec(portals() -> [{node(), map()}]). -portals() -> - [{Name, emqx_portal:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. +-spec(bridges() -> [{node(), map()}]). +bridges() -> + [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?SUP)]. -create_portal(Id, Config) -> - supervisor:start_child(?SUP, portal_spec({Id, Config})). +create_bridge(Id, Config) -> + supervisor:start_child(?SUP, bridge_spec({Id, Config})). -drop_portal(Id) -> +drop_bridge(Id) -> case supervisor:terminate_child(?SUP, Id) of ok -> supervisor:delete_child(?SUP, Id); diff --git a/src/emqx_portal_connect.erl b/src/emqx_bridge_connect.erl similarity index 94% rename from src/emqx_portal_connect.erl rename to src/emqx_bridge_connect.erl index 9c2d168f3..b2781cc2c 100644 --- a/src/emqx_portal_connect.erl +++ b/src/emqx_bridge_connect.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_connect). +-module(emqx_bridge_connect). -export([start/2]). @@ -25,7 +25,7 @@ -type connection() :: term(). -type conn_ref() :: term(). -type batch() :: emqx_protal:batch(). --type ack_ref() :: emqx_portal:ack_ref(). +-type ack_ref() :: emqx_bridge:ack_ref(). -type topic() :: emqx_topic:topic(). -type qos() :: emqx_mqtt_types:qos(). @@ -37,7 +37,7 @@ -callback start(config()) -> {ok, conn_ref(), connection()} | {error, any()}. %% send to remote node/cluster -%% portal worker (the caller process) should be expecting +%% bridge worker (the caller process) should be expecting %% a message {batch_ack, reference()} when batch is acknowledged by remote node/cluster -callback send(connection(), batch()) -> {ok, ack_ref()} | {error, any()}. diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 0e6ebb08a..eff33a841 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -61,7 +61,7 @@ init([]) -> RouterSup = supervisor_spec(emqx_router_sup), %% Broker Sup BrokerSup = supervisor_spec(emqx_broker_sup), - PortalSup = supervisor_spec(emqx_portal_sup), + BridgeSup = supervisor_spec(emqx_bridge_sup), %% AccessControl AccessControl = worker_spec(emqx_access_control), %% Session Manager @@ -74,7 +74,7 @@ init([]) -> [KernelSup, RouterSup, BrokerSup, - PortalSup, + BridgeSup, AccessControl, SMSup, CMSup, @@ -88,4 +88,3 @@ worker_spec(M) -> {M, {M, start_link, []}, permanent, 30000, worker, [M]}. supervisor_spec(M) -> {M, {M, start_link, []}, permanent, infinity, supervisor, [M]}. - diff --git a/test/emqx_portal_SUITE.erl b/test/emqx_bridge_SUITE.erl similarity index 79% rename from test/emqx_portal_SUITE.erl rename to test/emqx_bridge_SUITE.erl index 2ceb7ea1e..465b20637 100644 --- a/test/emqx_portal_SUITE.erl +++ b/test/emqx_bridge_SUITE.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_SUITE). +-module(emqx_bridge_SUITE). -export([all/0, init_per_suite/1, end_per_suite/1]). -export([t_rpc/1, @@ -48,39 +48,39 @@ t_mngr(Config) when is_list(Config) -> Subs = [{<<"a">>, 1}, {<<"b">>, 2}], Cfg = #{address => node(), forwards => [<<"mngr">>], - connect_module => emqx_portal_rpc, + connect_module => emqx_bridge_rpc, mountpoint => <<"forwarded">>, subscriptions => Subs, start_type => auto }, Name = ?FUNCTION_NAME, - {ok, Pid} = emqx_portal:start_link(Name, Cfg), + {ok, Pid} = emqx_bridge:start_link(Name, Cfg), try - ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Name)), - ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_portal:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_portal:get_forwards(Pid)), - ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_portal:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_portal:get_forwards(Pid)), + ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Name)), + ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr")), + ?assertEqual(ok, emqx_bridge:ensure_forward_present(Name, "mngr2")), + ?assertEqual([<<"mngr">>, <<"mngr2">>], emqx_bridge:get_forwards(Pid)), + ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr2")), + ?assertEqual(ok, emqx_bridge:ensure_forward_absent(Name, "mngr3")), + ?assertEqual([<<"mngr">>], emqx_bridge:get_forwards(Pid)), ?assertEqual({error, no_remote_subscription_support}, - emqx_portal:ensure_subscription_present(Pid, <<"t">>, 0)), + emqx_bridge:ensure_subscription_present(Pid, <<"t">>, 0)), ?assertEqual({error, no_remote_subscription_support}, - emqx_portal:ensure_subscription_absent(Pid, <<"t">>)), - ?assertEqual(Subs, emqx_portal:get_subscriptions(Pid)) + emqx_bridge:ensure_subscription_absent(Pid, <<"t">>)), + ?assertEqual(Subs, emqx_bridge:get_subscriptions(Pid)) after - ok = emqx_portal:stop(Pid) + ok = emqx_bridge:stop(Pid) end. %% A loopback RPC to local node t_rpc(Config) when is_list(Config) -> Cfg = #{address => node(), forwards => [<<"t_rpc/#">>], - connect_module => emqx_portal_rpc, + connect_module => emqx_bridge_rpc, mountpoint => <<"forwarded">>, start_type => auto }, - {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), + {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg), ClientId = <<"ClientId">>, try {ok, ConnPid} = emqx_mock_client:start_link(ClientId), @@ -96,13 +96,13 @@ t_rpc(Config) when is_list(Config) -> end, 4000), emqx_mock_client:close_session(ConnPid) after - ok = emqx_portal:stop(Pid) + ok = emqx_bridge:stop(Pid) end. %% Full data loopback flow explained: %% test-pid ---> mock-cleint ----> local-broker ---(local-subscription)---> -%% portal(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> -%% portal(import) --(mecked message sending)--> test-pid +%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> +%% bridge(import) --(mecked message sending)--> test-pid t_mqtt(Config) when is_list(Config) -> SendToTopic = <<"t_mqtt/one">>, SendToTopic2 = <<"t_mqtt/two">>, @@ -111,7 +111,7 @@ t_mqtt(Config) when is_list(Config) -> ForwardedTopic2 = emqx_topic:join(["forwarded", atom_to_list(node()), SendToTopic2]), Cfg = #{address => "127.0.0.1:1883", forwards => [SendToTopic], - connect_module => emqx_portal_mqtt, + connect_module => emqx_bridge_mqtt, mountpoint => Mountpoint, username => "user", clean_start => true, @@ -128,26 +128,26 @@ t_mqtt(Config) when is_list(Config) -> reconnect_delay_ms => 1000, ssl => false, %% Consume back to forwarded message for verification - %% NOTE: this is a indefenite loopback without mocking emqx_portal:import_batch/2 + %% NOTE: this is a indefenite loopback without mocking emqx_bridge:import_batch/2 subscriptions => [{ForwardedTopic, _QoS = 1}], start_type => auto }, Tester = self(), Ref = make_ref(), - meck:new(emqx_portal, [passthrough, no_history]), - meck:expect(emqx_portal, import_batch, 2, + meck:new(emqx_bridge, [passthrough, no_history]), + meck:expect(emqx_bridge, import_batch, 2, fun(Batch, AckFun) -> Tester ! {Ref, Batch}, AckFun() end), - {ok, Pid} = emqx_portal:start_link(?FUNCTION_NAME, Cfg), + {ok, Pid} = emqx_bridge:start_link(?FUNCTION_NAME, Cfg), ClientId = <<"client-1">>, try - ?assertEqual([{ForwardedTopic, 1}], emqx_portal:get_subscriptions(Pid)), - ok = emqx_portal:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), - ok = emqx_portal:ensure_forward_present(Pid, SendToTopic2), + ?assertEqual([{ForwardedTopic, 1}], emqx_bridge:get_subscriptions(Pid)), + ok = emqx_bridge:ensure_subscription_present(Pid, ForwardedTopic2, _QoS = 1), + ok = emqx_bridge:ensure_forward_present(Pid, SendToTopic2), ?assertEqual([{ForwardedTopic, 1}, - {ForwardedTopic2, 1}], emqx_portal:get_subscriptions(Pid)), + {ForwardedTopic2, 1}], emqx_bridge:get_subscriptions(Pid)), {ok, ConnPid} = emqx_mock_client:start_link(ClientId), {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), %% message from a different client, to avoid getting terminated by no-local @@ -166,8 +166,8 @@ t_mqtt(Config) when is_list(Config) -> ok = receive_and_match_messages(Ref, Msgs2), emqx_mock_client:close_session(ConnPid) after - ok = emqx_portal:stop(Pid), - meck:unload(emqx_portal) + ok = emqx_bridge:stop(Pid), + meck:unload(emqx_bridge) end. receive_and_match_messages(Ref, Msgs) -> diff --git a/test/emqx_portal_mqtt_tests.erl b/test/emqx_bridge_mqtt_tests.erl similarity index 91% rename from test/emqx_portal_mqtt_tests.erl rename to test/emqx_bridge_mqtt_tests.erl index 1fcd71e79..7c094b957 100644 --- a/test/emqx_portal_mqtt_tests.erl +++ b/test/emqx_bridge_mqtt_tests.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_mqtt_tests). +-module(emqx_bridge_mqtt_tests). -include_lib("eunit/include/eunit.hrl"). -include("emqx_mqtt.hrl"). @@ -39,12 +39,12 @@ send_and_ack_test() -> try Max = 100, Batch = lists:seq(1, Max), - {ok, Ref, Conn} = emqx_portal_mqtt:start(#{address => "127.0.0.1:1883"}), + {ok, Ref, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}), %% return last packet id as batch reference - {ok, AckRef} = emqx_portal_mqtt:send(Conn, Batch), + {ok, AckRef} = emqx_bridge_mqtt:send(Conn, Batch), %% expect batch ack receive {batch_ack, AckRef} -> ok end, - ok = emqx_portal_mqtt:stop(Ref, Conn) + ok = emqx_bridge_mqtt:stop(Ref, Conn) after meck:unload(emqx_client) end. diff --git a/test/emqx_portal_rpc_tests.erl b/test/emqx_bridge_rpc_tests.erl similarity index 80% rename from test/emqx_portal_rpc_tests.erl rename to test/emqx_bridge_rpc_tests.erl index 363a0ee5a..28e05b895 100644 --- a/test/emqx_portal_rpc_tests.erl +++ b/test/emqx_bridge_rpc_tests.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_rpc_tests). +-module(emqx_bridge_rpc_tests). -include_lib("eunit/include/eunit.hrl"). send_and_ack_test() -> @@ -26,18 +26,18 @@ send_and_ack_test() -> fun(Node, Module, Fun, Args) -> rpc:cast(Node, Module, Fun, Args) end), - meck:new(emqx_portal, [passthrough, no_history]), - meck:expect(emqx_portal, import_batch, 2, + meck:new(emqx_bridge, [passthrough, no_history]), + meck:expect(emqx_bridge, import_batch, 2, fun(batch, AckFun) -> AckFun() end), try - {ok, Pid, Node} = emqx_portal_rpc:start(#{address => node()}), - {ok, Ref} = emqx_portal_rpc:send(Node, batch), + {ok, Pid, Node} = emqx_bridge_rpc:start(#{address => node()}), + {ok, Ref} = emqx_bridge_rpc:send(Node, batch), receive {batch_ack, Ref} -> ok end, - ok = emqx_portal_rpc:stop(Pid, Node) + ok = emqx_bridge_rpc:stop(Pid, Node) after meck:unload(gen_rpc), - meck:unload(emqx_portal) + meck:unload(emqx_bridge) end. diff --git a/test/emqx_portal_tests.erl b/test/emqx_bridge_tests.erl similarity index 75% rename from test/emqx_portal_tests.erl rename to test/emqx_bridge_tests.erl index 7cca66adc..22b2c4d49 100644 --- a/test/emqx_portal_tests.erl +++ b/test/emqx_bridge_tests.erl @@ -12,15 +12,15 @@ %% See the License for the specific language governing permissions and %% limitations under the License. --module(emqx_portal_tests). --behaviour(emqx_portal_connect). +-module(emqx_bridge_tests). +-behaviour(emqx_bridge_connect). -include_lib("eunit/include/eunit.hrl"). -include("emqx.hrl"). -include("emqx_mqtt.hrl"). --define(PORTAL_NAME, test). --define(PORTAL_REG_NAME, emqx_portal_test). +-define(BRIDGE_NAME, test). +-define(BRIDGE_REG_NAME, emqx_bridge_test). -define(WAIT(PATTERN, TIMEOUT), receive PATTERN -> @@ -45,29 +45,29 @@ send(SendFun, Batch) when is_function(SendFun, 1) -> stop(_Ref, _Pid) -> ok. -%% portal worker should retry connecting remote node indefinitely +%% bridge worker should retry connecting remote node indefinitely reconnect_test() -> Ref = make_ref(), Config = make_config(Ref, self(), {error, test}), - {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), %% assert name registered - ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), ?WAIT({connection_start_attempt, Ref}, 1000), %% expect same message again ?WAIT({connection_start_attempt, Ref}, 1000), - ok = emqx_portal:stop(?PORTAL_REG_NAME), + ok = emqx_bridge:stop(?BRIDGE_REG_NAME), ok. %% connect first, disconnect, then connect again disturbance_test() -> Ref = make_ref(), Config = make_config(Ref, self(), {ok, Ref, connection}), - {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), - ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), ?WAIT({connection_start_attempt, Ref}, 1000), Pid ! {disconnected, Ref, test}, ?WAIT({connection_start_attempt, Ref}, 1000), - ok = emqx_portal:stop(?PORTAL_REG_NAME). + ok = emqx_bridge:stop(?BRIDGE_REG_NAME). %% buffer should continue taking in messages when disconnected buffer_when_disconnected_test_() -> @@ -76,9 +76,9 @@ buffer_when_disconnected_test_() -> test_buffer_when_disconnected() -> Ref = make_ref(), Nums = lists:seq(1, 100), - Sender = spawn_link(fun() -> receive {portal, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end), + Sender = spawn_link(fun() -> receive {bridge, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end), SenderMref = monitor(process, Sender), - Receiver = spawn_link(fun() -> receive {portal, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end), + Receiver = spawn_link(fun() -> receive {bridge, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end), ReceiverMref = monitor(process, Receiver), SendFun = fun(Batch) -> BatchRef = make_ref(), @@ -87,44 +87,44 @@ test_buffer_when_disconnected() -> end, Config0 = make_config(Ref, false, {ok, Ref, SendFun}), Config = Config0#{reconnect_delay_ms => 100}, - {ok, Pid} = emqx_portal:start_link(?PORTAL_NAME, Config), - Sender ! {portal, Pid}, - Receiver ! {portal, Pid}, - ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), + {ok, Pid} = emqx_bridge:start_link(?BRIDGE_NAME, Config), + Sender ! {bridge, Pid}, + Receiver ! {bridge, Pid}, + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), Pid ! {disconnected, Ref, test}, ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000), ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), - ok = emqx_portal:stop(?PORTAL_REG_NAME). + ok = emqx_bridge:stop(?BRIDGE_REG_NAME). manual_start_stop_test() -> Ref = make_ref(), Config0 = make_config(Ref, self(), {ok, Ref, connection}), Config = Config0#{start_type := manual}, - {ok, Pid} = emqx_portal:ensure_started(?PORTAL_NAME, Config), + {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config), %% call ensure_started again should yeld the same result - {ok, Pid} = emqx_portal:ensure_started(?PORTAL_NAME, Config), - ?assertEqual(Pid, whereis(?PORTAL_REG_NAME)), + {ok, Pid} = emqx_bridge:ensure_started(?BRIDGE_NAME, Config), + ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), ?assertEqual({error, standing_by}, - emqx_portal:ensure_forward_present(Pid, "dummy")), - emqx_portal:ensure_stopped(unknown), - emqx_portal:ensure_stopped(Pid), - emqx_portal:ensure_stopped(?PORTAL_REG_NAME). + emqx_bridge:ensure_forward_present(Pid, "dummy")), + emqx_bridge:ensure_stopped(unknown), + emqx_bridge:ensure_stopped(Pid), + emqx_bridge:ensure_stopped(?BRIDGE_REG_NAME). -%% Feed messages to portal +%% Feed messages to bridge sender_loop(_Pid, [], _) -> exit(normal); sender_loop(Pid, [Num | Rest], Interval) -> random_sleep(Interval), Pid ! {dispatch, dummy, make_msg(Num)}, sender_loop(Pid, Rest, Interval). -%% Feed acknowledgments to portal +%% Feed acknowledgments to bridge receiver_loop(_Pid, [], _) -> ok; receiver_loop(Pid, Nums, Interval) -> receive {batch, BatchRef, Batch} -> Rest = match_nums(Batch, Nums), random_sleep(Interval), - emqx_portal:handle_ack(Pid, BatchRef), + emqx_bridge:handle_ack(Pid, BatchRef), receiver_loop(Pid, Rest, Interval) end. diff --git a/test/emqx_ct_broker_helpers.erl b/test/emqx_ct_broker_helpers.erl index 0e0bfa3a4..88240be85 100644 --- a/test/emqx_ct_broker_helpers.erl +++ b/test/emqx_ct_broker_helpers.erl @@ -170,13 +170,13 @@ flush(Msgs) -> bridge_conf() -> [ {local_rpc, - [{connect_module, emqx_portal_rpc}, + [{connect_module, emqx_bridge_rpc}, {address, node()}, - {forwards, ["portal-1/#", "portal-2/#"]} + {forwards, ["bridge-1/#", "bridge-2/#"]} ]} ]. % [{aws, - % [{connect_module, emqx_portal_mqtt}, + % [{connect_module, emqx_bridge_mqtt}, % {username,"user"}, % {address,"127.0.0.1:1883"}, % {clean_start,true}, From 055d0ad98e846dffd3f5143a494621aba210da80 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Thu, 28 Feb 2019 15:39:11 +0800 Subject: [PATCH 26/26] Fix typo --- src/emqx_client.erl | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/emqx_client.erl b/src/emqx_client.erl index 2b551be56..e29e50552 100644 --- a/src/emqx_client.erl +++ b/src/emqx_client.erl @@ -789,21 +789,21 @@ connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) -> connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, State = #state{inflight = Inflight, last_packet_id = PacketId}) - when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> - case emqx_inflight:is_full(Inflight) of - true -> - {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]}; - false -> - Msg1 = Msg#mqtt_msg{packet_id = PacketId}, - case send(Msg1, State) of - {ok, NewState} -> - Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), - {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), - [{reply, From, {ok, PacketId}}]}; - {error, Reason} -> - {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} - end - end; + when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> + case emqx_inflight:is_full(Inflight) of + true -> + {keep_state, State, [{reply, From, {error, {PacketId, inflight_full}}}]}; + false -> + Msg1 = Msg#mqtt_msg{packet_id = PacketId}, + case send(Msg1, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), + {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), + [{reply, From, {ok, PacketId}}]}; + {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, {error, {PacketId, Reason}}}]} + end + end; connected({call, From}, UnsubReq = {unsubscribe, Properties, Topics}, State = #state{last_packet_id = PacketId}) ->