From 4da0d83faf3e1759d74024427c21fd859a5c4760 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 18 May 2023 20:44:20 +0300 Subject: [PATCH 01/17] chore(mqttconn): remove dead code --- .../src/emqx_connector_mqtt.erl | 52 +------------------ .../src/mqtt/emqx_connector_mqtt_msg.erl | 24 +-------- 2 files changed, 2 insertions(+), 74 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index bb8cc00d1..1efe6f3e0 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -15,10 +15,6 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mqtt). --include("emqx_connector.hrl"). - --include_lib("typerefl/include/types.hrl"). --include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(supervisor). @@ -47,52 +43,6 @@ -export([on_async_result/2]). --behaviour(hocon_schema). - --import(hoconsc, [mk/2]). - --export([ - roots/0, - fields/1 -]). - -%%===================================================================== -%% Hocon schema -roots() -> - fields("config"). - -fields("config") -> - emqx_connector_mqtt_schema:fields("config"); -fields("get") -> - [ - {num_of_bridges, - mk( - integer(), - #{desc => ?DESC("num_of_bridges")} - )} - ] ++ fields("post"); -fields("put") -> - emqx_connector_mqtt_schema:fields("server_configs"); -fields("post") -> - [ - {type, - mk( - mqtt, - #{ - required => true, - desc => ?DESC("type") - } - )}, - {name, - mk( - binary(), - #{ - required => true, - desc => ?DESC("name") - } - )} - ] ++ fields("put"). - %% =================================================================== %% supervisor APIs start_link() -> @@ -313,7 +263,7 @@ maybe_put_fields(Fields, Conf, Acc0) -> ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Id, _Conf = #{clientid_prefix := Prefix = <<_/binary>>}) -> +clientid(Id, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) -> iolist_to_binary([Prefix, ":", Id, ":", atom_to_list(node())]); clientid(Id, _Conf) -> iolist_to_binary([Id, ":", atom_to_list(node())]). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index df1114483..8fc70405f 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -17,12 +17,9 @@ -module(emqx_connector_mqtt_msg). -export([ - to_binary/1, - from_binary/1, make_pub_vars/2, to_remote_msg/2, - to_broker_msg/3, - estimate_size/1 + to_broker_msg/3 ]). -export([ @@ -143,25 +140,6 @@ replace_simple_var(Tokens, Data) when is_list(Tokens) -> replace_simple_var(Val, _Data) -> Val. -%% @doc Make `binary()' in order to make iodata to be persisted on disk. --spec to_binary(msg()) -> binary(). -to_binary(Msg) -> term_to_binary(Msg). - -%% @doc Unmarshal binary into `msg()'. --spec from_binary(binary()) -> msg(). -from_binary(Bin) -> binary_to_term(Bin). - -%% @doc Estimate the size of a message. -%% Count only the topic length + payload size -%% There is no topic and payload for event message. So count all `Msg` term --spec estimate_size(msg()) -> integer(). -estimate_size(#message{topic = Topic, payload = Payload}) -> - size(Topic) + size(Payload); -estimate_size(#{topic := Topic, payload := Payload}) -> - size(Topic) + size(Payload); -estimate_size(Term) -> - erlang:external_size(Term). - set_headers(Val, Msg) -> emqx_message:set_headers(Val, Msg). topic(undefined, Topic) -> Topic; From bd956d00b6aa5fd21140473f01f2a4f188377811 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 18 May 2023 20:53:42 +0300 Subject: [PATCH 02/17] feat(mqttconn): stop using gproc in hot path Also drop fiddling with `mountpoint` since this option seems not to be used anywhere. --- .../test/emqx_bridge_mqtt_SUITE.erl | 4 +- .../src/emqx_connector_mqtt.erl | 124 +++----- .../src/mqtt/emqx_connector_mqtt_msg.erl | 22 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 268 +++++++++--------- 4 files changed, 186 insertions(+), 232 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index f0de07da2..c00eb6b14 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -256,11 +256,11 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> } ), - {ok, _, #{state := #{name := WorkerName}}} = + {ok, _, #{state := #{worker := WorkerPid}}} = emqx_resource:get_instance(emqx_bridge_resource:resource_id(BridgeID)), ?assertMatch( #{clean_start := true}, - maps:from_list(emqx_connector_mqtt_worker:info(WorkerName)) + maps:from_list(emqx_connector_mqtt_worker:info(WorkerPid)) ), %% delete the bridge diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 1efe6f3e0..bd4bf6eb1 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -25,8 +25,8 @@ callback_mode/0, start_link/0, init/1, - create_bridge/1, - drop_bridge/1, + create_bridge/2, + remove_bridge/1, bridges/0 ]). @@ -56,11 +56,10 @@ init([]) -> }, {ok, {SupFlag, []}}. -bridge_spec(Config) -> - {Name, NConfig} = maps:take(name, Config), +bridge_spec(Name, Options) -> #{ id => Name, - start => {emqx_connector_mqtt_worker, start_link, [Name, NConfig]}, + start => {emqx_connector_mqtt_worker, start_link, [Name, Options]}, restart => temporary, shutdown => 1000 }. @@ -72,10 +71,10 @@ bridges() -> || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE) ]. -create_bridge(Config) -> - supervisor:start_child(?MODULE, bridge_spec(Config)). +create_bridge(Name, Options) -> + supervisor:start_child(?MODULE, bridge_spec(Name, Options)). -drop_bridge(Name) -> +remove_bridge(Name) -> case supervisor:terminate_child(?MODULE, Name) of ok -> supervisor:delete_child(?MODULE, Name); @@ -95,36 +94,39 @@ on_message_received(Msg, HookPoint, ResId) -> %% =================================================================== callback_mode() -> async_if_possible. -on_start(InstanceId, Conf) -> +on_start(ResourceId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", - connector => InstanceId, + connector => ResourceId, config => emqx_utils:redact(Conf) }), BasicConf = basic_config(Conf), - BridgeConf = BasicConf#{ - name => InstanceId, - clientid => clientid(InstanceId, Conf), - subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), Conf, InstanceId), - forwards => make_forward_confs(maps:get(egress, Conf, undefined)) + BridgeOpts = BasicConf#{ + clientid => clientid(ResourceId, Conf), + subscriptions => make_sub_confs(maps:get(ingress, Conf, #{}), Conf, ResourceId), + forwards => maps:get(egress, Conf, #{}) }, - case ?MODULE:create_bridge(BridgeConf) of - {ok, _Pid} -> - ensure_mqtt_worker_started(InstanceId, BridgeConf); + case create_bridge(ResourceId, BridgeOpts) of + {ok, Pid, {ConnProps, WorkerConf}} -> + {ok, #{ + name => ResourceId, + worker => Pid, + config => WorkerConf, + props => ConnProps + }}; {error, {already_started, _Pid}} -> - ok = ?MODULE:drop_bridge(InstanceId), - {ok, _} = ?MODULE:create_bridge(BridgeConf), - ensure_mqtt_worker_started(InstanceId, BridgeConf); + ok = remove_bridge(ResourceId), + on_start(ResourceId, Conf); {error, Reason} -> {error, Reason} end. -on_stop(_InstId, #{name := InstanceId}) -> +on_stop(ResourceId, #{}) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", - connector => InstanceId + connector => ResourceId }), - case ?MODULE:drop_bridge(InstanceId) of + case remove_bridge(ResourceId) of ok -> ok; {error, not_found} -> @@ -132,24 +134,24 @@ on_stop(_InstId, #{name := InstanceId}) -> {error, Reason} -> ?SLOG(error, #{ msg => "stop_mqtt_connector_error", - connector => InstanceId, + connector => ResourceId, reason => Reason }) end. -on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) -> - ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), - case emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg) of +on_query(ResourceId, {send_message, Msg}, #{worker := Pid, config := Config}) -> + ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), + case emqx_connector_mqtt_worker:send_to_remote(Pid, Msg, Config) of ok -> ok; {error, Reason} -> classify_error(Reason) end. -on_query_async(_InstId, {send_message, Msg}, CallbackIn, #{name := InstanceId}) -> - ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), +on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{worker := Pid, config := Config}) -> + ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - case emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, Callback) of + case emqx_connector_mqtt_worker:send_to_remote_async(Pid, Msg, Callback, Config) of ok -> ok; {ok, Pid} -> @@ -172,8 +174,8 @@ apply_callback_function({F, A}, Result) when is_function(F), is_list(A) -> apply_callback_function({M, F, A}, Result) when is_atom(M), is_atom(F), is_list(A) -> erlang:apply(M, F, A ++ [Result]). -on_get_status(_InstId, #{name := InstanceId}) -> - emqx_connector_mqtt_worker:status(InstanceId). +on_get_status(_ResourceId, #{worker := Pid}) -> + emqx_connector_mqtt_worker:status(Pid). classify_error(disconnected = Reason) -> {error, {recoverable_error, Reason}}; @@ -186,33 +188,13 @@ classify_error(shutdown = Reason) -> classify_error(Reason) -> {error, {unrecoverable_error, Reason}}. -ensure_mqtt_worker_started(InstanceId, BridgeConf) -> - case emqx_connector_mqtt_worker:connect(InstanceId) of - {ok, Properties} -> - {ok, #{name => InstanceId, config => BridgeConf, props => Properties}}; - {error, Reason} -> - {error, Reason} - end. - -make_sub_confs(EmptyMap, _Conf, _) when map_size(EmptyMap) == 0 -> - undefined; -make_sub_confs(undefined, _Conf, _) -> - undefined; -make_sub_confs(SubRemoteConf, Conf, ResourceId) -> - case maps:find(hookpoint, Conf) of - error -> - error({no_hookpoint_provided, Conf}); - {ok, HookPoint} -> - MFA = {?MODULE, on_message_received, [HookPoint, ResourceId]}, - SubRemoteConf#{on_message_received => MFA} - end. - -make_forward_confs(EmptyMap) when map_size(EmptyMap) == 0 -> - undefined; -make_forward_confs(undefined) -> - undefined; -make_forward_confs(FrowardConf) -> - FrowardConf. +make_sub_confs(Subscriptions, _Conf, _) when map_size(Subscriptions) == 0 -> + Subscriptions; +make_sub_confs(Subscriptions, #{hookpoint := HookPoint}, ResourceId) -> + MFA = {?MODULE, on_message_received, [HookPoint, ResourceId]}, + Subscriptions#{on_message_received => MFA}; +make_sub_confs(_SubRemoteConf, Conf, ResourceId) -> + error({no_hookpoint_provided, ResourceId, Conf}). basic_config( #{ @@ -227,17 +209,14 @@ basic_config( } = Conf ) -> BasicConf = #{ - %% connection opts server => Server, %% 30s connect_timeout => 30, - auto_reconnect => true, proto_ver => ProtoVer, - %% Opening bridge_mode will form a non-standard mqtt connection message. + %% Opening a connection in bridge mode will form a non-standard mqtt connection message. %% A load balancing server (such as haproxy) is often set up before the emqx broker server. %% When the load balancing server enables mqtt connection packet inspection, - %% non-standard mqtt connection packets will be filtered out by LB. - %% So let's disable bridge_mode. + %% non-standard mqtt connection packets might be filtered out by LB. bridge_mode => BridgeMode, keepalive => ms_to_s(KeepAlive), clean_start => CleanStart, @@ -246,18 +225,9 @@ basic_config( ssl => EnableSsl, ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }, - maybe_put_fields([username, password], Conf, BasicConf). - -maybe_put_fields(Fields, Conf, Acc0) -> - lists:foldl( - fun(Key, Acc) -> - case maps:find(Key, Conf) of - error -> Acc; - {ok, Val} -> Acc#{Key => Val} - end - end, - Acc0, - Fields + maps:merge( + BasicConf, + maps:with([username, password], Conf) ). ms_to_s(Ms) -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 8fc70405f..004819678 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -17,7 +17,6 @@ -module(emqx_connector_mqtt_msg). -export([ - make_pub_vars/2, to_remote_msg/2, to_broker_msg/3 ]). @@ -46,11 +45,6 @@ remote := remote_config() }. -make_pub_vars(_, undefined) -> - undefined; -make_pub_vars(Mountpoint, Conf) when is_map(Conf) -> - Conf#{mountpoint => Mountpoint}. - %% @doc Make export format: %% 1. Mount topic to a prefix %% 2. Fix QoS to 1 @@ -70,8 +64,7 @@ to_remote_msg(MapMsg, #{ topic := TopicToken, qos := QoSToken, retain := RetainToken - } = Remote, - mountpoint := Mountpoint + } = Remote }) when is_map(MapMsg) -> Topic = replace_vars_in_str(TopicToken, MapMsg), Payload = process_payload(Remote, MapMsg), @@ -81,12 +74,10 @@ to_remote_msg(MapMsg, #{ #mqtt_msg{ qos = QoS, retain = Retain, - topic = topic(Mountpoint, Topic), + topic = Topic, props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload - }; -to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> - Msg#message{topic = topic(Mountpoint, Topic)}. + }. %% published from remote node over a MQTT connection to_broker_msg(Msg, Vars, undefined) -> @@ -98,8 +89,7 @@ to_broker_msg( topic := TopicToken, qos := QoSToken, retain := RetainToken - } = Local, - mountpoint := Mountpoint + } = Local }, Props ) -> @@ -112,7 +102,7 @@ to_broker_msg( Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, emqx_message:set_flags( #{dup => Dup, retain => Retain}, - emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload) + emqx_message:make(bridge, QoS, Topic, Payload) ) ). @@ -142,5 +132,3 @@ replace_simple_var(Val, _Data) -> set_headers(Val, Msg) -> emqx_message:set_headers(Val, Msg). -topic(undefined, Topic) -> Topic; -topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index e49603e51..c49d5b180 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -72,34 +72,75 @@ %% management APIs -export([ - connect/1, status/1, ping/1, info/1, - send_to_remote/2, - send_to_remote_async/3 + send_to_remote/3, + send_to_remote_async/4 ]). -export([handle_publish/3]). -export([handle_disconnect/1]). --export_type([ - config/0, - ack_ref/0 -]). +-export_type([config/0]). + +-type template() :: emqx_plugin_libs_rule:tmpl_token(). -type name() :: term(). -% -type qos() :: emqx_types:qos(). --type config() :: map(). --type ack_ref() :: term(). -% -type topic() :: emqx_types:topic(). +-type options() :: #{ + % endpoint + server := iodata(), + % emqtt client options + proto_ver := v3 | v4 | v5, + username := binary(), + password := binary(), + clientid := binary(), + clean_start := boolean(), + max_inflight := pos_integer(), + connect_timeout := pos_integer(), + retry_interval := timeout(), + bridge_mode := boolean(), + ssl := boolean(), + ssl_opts := proplists:proplist(), + % bridge options + subscriptions := map(), + forwards := map() +}. + +-type config() :: #{ + subscriptions := subscriptions() | undefined, + forwards := forwards() | undefined +}. + +-type subscriptions() :: #{ + remote := #{ + topic := emqx_topic:topic(), + qos => emqx_types:qos() + }, + local := #{ + topic => template(), + qos => template() | emqx_types:qos(), + retain => template() | boolean(), + payload => template() | undefined + }, + on_message_received := {module(), atom(), [term()]} +}. + +-type forwards() :: #{ + local => #{ + topic => emqx_topic:topic() + }, + remote := #{ + topic := template(), + qos => template() | emqx_types:qos(), + retain => template() | boolean(), + payload => template() | undefined + } +}. -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --define(REF(Name), {via, gproc, ?NAME(Name)}). --define(NAME(Name), {n, l, Name}). - %% @doc Start a bridge worker. Supported configs: %% mountpoint: The topic mount point for messages sent to remote node/cluster %% `undefined', `<<>>' or `""' to disable @@ -107,20 +148,19 @@ %% %% Find more connection specific configs in the callback modules %% of emqx_bridge_connect behaviour. --spec start_link(name(), map()) -> - {ok, pid()} | {error, _Reason}. +-spec start_link(name(), options()) -> + {ok, pid(), {emqtt:properties(), config()}} | {error, _Reason}. start_link(Name, BridgeOpts) -> ?SLOG(debug, #{ msg => "client_starting", name => Name, options => BridgeOpts }), - Conf = init_config(Name, BridgeOpts), - Options = mk_client_options(Conf, BridgeOpts), + Config = init_config(Name, BridgeOpts), + Options = mk_client_options(Config, BridgeOpts), case emqtt:start_link(Options) of {ok, Pid} -> - true = gproc:reg_other(?NAME(Name), Pid, Conf), - {ok, Pid}; + connect(Pid, Name, Config); {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", @@ -130,22 +170,50 @@ start_link(Name, BridgeOpts) -> Error end. +connect(Pid, Name, Config = #{subscriptions := Subscriptions}) -> + case emqtt:connect(Pid) of + {ok, Props} -> + case subscribe_remote_topics(Pid, Subscriptions) of + ok -> + {ok, Pid, {Props, Config}}; + {ok, _, _RCs} -> + {ok, Pid, {Props, Config}}; + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "client_subscribe_failed", + subscriptions => Subscriptions, + reason => Reason + }), + _ = emqtt:stop(Pid), + Error + end; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "client_connect_failed", + reason => Reason, + name => Name + }), + _ = emqtt:stop(Pid), + Error + end. + +subscribe_remote_topics(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> + emqtt:subscribe(Pid, RemoteTopic, QoS); +subscribe_remote_topics(_Ref, undefined) -> + ok. + init_config(Name, Opts) -> - Mountpoint = maps:get(forward_mountpoint, Opts, undefined), Subscriptions = maps:get(subscriptions, Opts, undefined), Forwards = maps:get(forwards, Opts, undefined), #{ - mountpoint => format_mountpoint(Mountpoint), subscriptions => pre_process_subscriptions(Subscriptions, Name, Opts), forwards => pre_process_forwards(Forwards) }. -mk_client_options(Conf, BridgeOpts) -> +mk_client_options(Config, BridgeOpts) -> Server = iolist_to_binary(maps:get(server, BridgeOpts)), HostPort = emqx_connector_mqtt_schema:parse_server(Server), - Mountpoint = maps:get(receive_mountpoint, BridgeOpts, undefined), - Subscriptions = maps:get(subscriptions, Conf), - Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Subscriptions), + Subscriptions = maps:get(subscriptions, Config), CleanStart = case Subscriptions of #{remote := _} -> @@ -156,24 +224,26 @@ mk_client_options(Conf, BridgeOpts) -> %% to ensure proper session recovery according to the MQTT spec. true end, - Opts = maps:without( + Opts = maps:with( [ - address, - auto_reconnect, - conn_type, - mountpoint, - forwards, - receive_mountpoint, - subscriptions + proto_ver, + username, + password, + clientid, + max_inflight, + connect_timeout, + retry_interval, + bridge_mode, + ssl, + ssl_opts ], BridgeOpts ), Opts#{ - msg_handler => mk_client_event_handler(Vars, #{server => Server}), + msg_handler => mk_client_event_handler(Subscriptions, #{server => Server}), hosts => [HostPort], clean_start => CleanStart, - force_ping => true, - proto_ver => maps:get(proto_ver, BridgeOpts, v4) + force_ping => true }. mk_client_event_handler(Vars, Opts) when Vars /= undefined -> @@ -184,45 +254,15 @@ mk_client_event_handler(Vars, Opts) when Vars /= undefined -> mk_client_event_handler(undefined, _Opts) -> undefined. -connect(Name) -> - #{subscriptions := Subscriptions} = get_config(Name), - case emqtt:connect(get_pid(Name)) of - {ok, Properties} -> - case subscribe_remote_topics(Name, Subscriptions) of - ok -> - {ok, Properties}; - {ok, _, _RCs} -> - {ok, Properties}; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "client_subscribe_failed", - subscriptions => Subscriptions, - reason => Reason - }), - Error - end; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "client_connect_failed", - reason => Reason, - name => Name - }), - Error - end. -subscribe_remote_topics(Ref, #{remote := #{topic := FromTopic, qos := QoS}}) -> - emqtt:subscribe(ref(Ref), FromTopic, QoS); -subscribe_remote_topics(_Ref, undefined) -> - ok. +stop(Pid) -> + emqtt:stop(Pid). -stop(Ref) -> - emqtt:stop(ref(Ref)). +info(Pid) -> + emqtt:info(Pid). -info(Ref) -> - emqtt:info(ref(Ref)). - -status(Ref) -> +status(Pid) -> try - case proplists:get_value(socket, info(Ref)) of + case proplists:get_value(socket, info(Pid)) of Socket when Socket /= undefined -> connected; undefined -> @@ -233,14 +273,14 @@ status(Ref) -> disconnected end. -ping(Ref) -> - emqtt:ping(ref(Ref)). +ping(Pid) -> + emqtt:ping(Pid). -send_to_remote(Name, MsgIn) -> - trycall(fun() -> do_send(Name, export_msg(Name, MsgIn)) end). +send_to_remote(Pid, MsgIn, Conf) -> + do_send(Pid, export_msg(MsgIn, Conf)). -do_send(Name, {true, Msg}) -> - case emqtt:publish(get_pid(Name), Msg) of +do_send(Pid, {true, Msg}) -> + case emqtt:publish(Pid, Msg) of ok -> ok; {ok, #{reason_code := RC}} when @@ -266,36 +306,15 @@ do_send(Name, {true, Msg}) -> do_send(_Name, false) -> ok. -send_to_remote_async(Name, MsgIn, Callback) -> - trycall(fun() -> do_send_async(Name, export_msg(Name, MsgIn), Callback) end). +send_to_remote_async(Pid, MsgIn, Callback, Conf) -> + do_send_async(Pid, export_msg(MsgIn, Conf), Callback). -do_send_async(Name, {true, Msg}, Callback) -> - Pid = get_pid(Name), +do_send_async(Pid, {true, Msg}, Callback) -> ok = emqtt:publish_async(Pid, Msg, _Timeout = infinity, Callback), {ok, Pid}; -do_send_async(_Name, false, _Callback) -> +do_send_async(_Pid, false, _Callback) -> ok. -ref(Pid) when is_pid(Pid) -> - Pid; -ref(Term) -> - ?REF(Term). - -trycall(Fun) -> - try - Fun() - catch - throw:noproc -> - {error, disconnected}; - exit:{noproc, _} -> - {error, disconnected} - end. - -format_mountpoint(undefined) -> - undefined; -format_mountpoint(Prefix) -> - binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). - pre_process_subscriptions(undefined, _, _) -> undefined; pre_process_subscriptions( @@ -356,38 +375,15 @@ downgrade_ingress_qos(2) -> downgrade_ingress_qos(QoS) -> QoS. -get_pid(Name) -> - case gproc:where(?NAME(Name)) of - Pid when is_pid(Pid) -> - Pid; - undefined -> - throw(noproc) - end. - -get_config(Name) -> - try - gproc:lookup_value(?NAME(Name)) - catch - error:badarg -> - throw(noproc) - end. - -export_msg(Name, Msg) -> - case get_config(Name) of - #{forwards := Forwards = #{}, mountpoint := Mountpoint} -> - {true, export_msg(Mountpoint, Forwards, Msg)}; - #{forwards := undefined} -> - ?SLOG(error, #{ - msg => "forwarding_unavailable", - message => Msg, - reason => "egress is not configured" - }), - false - end. - -export_msg(Mountpoint, Forwards, Msg) -> - Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards), - emqx_connector_mqtt_msg:to_remote_msg(Msg, Vars). +export_msg(Msg, #{forwards := Forwards = #{}}) -> + {true, emqx_connector_mqtt_msg:to_remote_msg(Msg, Forwards)}; +export_msg(Msg, #{forwards := undefined}) -> + ?SLOG(error, #{ + msg => "forwarding_unavailable", + message => Msg, + reason => "egress is not configured" + }), + false. %% From 67d703f8c579c324ff33aa7cca6bc8b3143dbe76 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 19 May 2023 16:32:22 +0300 Subject: [PATCH 03/17] refactor(mqttconn): simplify mqtt connector Inline `emqx_connector_mqtt_msg` module code into `emqx_connector_mqtt_worker` module, since it's not really used anywhere else and does not provide any reusable abstractions. --- .../src/mqtt/emqx_connector_mqtt_msg.erl | 134 -------------- .../src/mqtt/emqx_connector_mqtt_worker.erl | 167 +++++++++++++----- 2 files changed, 122 insertions(+), 179 deletions(-) delete mode 100644 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl deleted file mode 100644 index 004819678..000000000 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ /dev/null @@ -1,134 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_mqtt_msg). - --export([ - to_remote_msg/2, - to_broker_msg/3 -]). - --export([ - replace_vars_in_str/2, - replace_simple_var/2 -]). - --export_type([msg/0]). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqtt/include/emqtt.hrl"). - --type msg() :: emqx_types:message(). --type exp_msg() :: emqx_types:message() | #mqtt_msg{}. --type remote_config() :: #{ - topic := binary(), - qos := original | integer(), - retain := original | boolean(), - payload := binary() -}. --type variables() :: #{ - mountpoint := undefined | binary(), - remote := remote_config() -}. - -%% @doc Make export format: -%% 1. Mount topic to a prefix -%% 2. Fix QoS to 1 -%% @end -%% Shame that we have to know the callback module here -%% would be great if we can get rid of #mqtt_msg{} record -%% and use #message{} in all places. --spec to_remote_msg(msg() | map(), variables()) -> - exp_msg(). -to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> - Retain0 = maps:get(retain, Flags0, false), - {Columns, _} = emqx_rule_events:eventmsg_publish(Msg), - MapMsg = maps:put(retain, Retain0, Columns), - to_remote_msg(MapMsg, Vars); -to_remote_msg(MapMsg, #{ - remote := #{ - topic := TopicToken, - qos := QoSToken, - retain := RetainToken - } = Remote -}) when is_map(MapMsg) -> - Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = process_payload(Remote, MapMsg), - QoS = replace_simple_var(QoSToken, MapMsg), - Retain = replace_simple_var(RetainToken, MapMsg), - PubProps = maps:get(pub_props, MapMsg, #{}), - #mqtt_msg{ - qos = QoS, - retain = Retain, - topic = Topic, - props = emqx_utils:pub_props_to_packet(PubProps), - payload = Payload - }. - -%% published from remote node over a MQTT connection -to_broker_msg(Msg, Vars, undefined) -> - to_broker_msg(Msg, Vars, #{}); -to_broker_msg( - #{dup := Dup} = MapMsg, - #{ - local := #{ - topic := TopicToken, - qos := QoSToken, - retain := RetainToken - } = Local - }, - Props -) -> - Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = process_payload(Local, MapMsg), - QoS = replace_simple_var(QoSToken, MapMsg), - Retain = replace_simple_var(RetainToken, MapMsg), - PubProps = maps:get(pub_props, MapMsg, #{}), - set_headers( - Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, - emqx_message:set_flags( - #{dup => Dup, retain => Retain}, - emqx_message:make(bridge, QoS, Topic, Payload) - ) - ). - -process_payload(From, MapMsg) -> - do_process_payload(maps:get(payload, From, undefined), MapMsg). - -do_process_payload(undefined, Msg) -> - emqx_utils_json:encode(Msg); -do_process_payload(Tks, Msg) -> - replace_vars_in_str(Tks, Msg). - -%% Replace a string contains vars to another string in which the placeholders are replace by the -%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: -%% "a: 1". -replace_vars_in_str(Tokens, Data) when is_list(Tokens) -> - emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary}); -replace_vars_in_str(Val, _Data) -> - Val. - -%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result -%% value will be an integer 1. -replace_simple_var(Tokens, Data) when is_list(Tokens) -> - [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), - Var; -replace_simple_var(Val, _Data) -> - Val. - -set_headers(Val, Msg) -> - emqx_message:set_headers(Val, Msg). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index c49d5b180..4169c7f69 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -61,8 +61,8 @@ -module(emqx_connector_mqtt_worker). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). %% APIs -export([ @@ -79,7 +79,7 @@ send_to_remote_async/4 ]). --export([handle_publish/3]). +-export([handle_publish/4]). -export([handle_disconnect/1]). -export_type([config/0]). @@ -117,12 +117,7 @@ topic := emqx_topic:topic(), qos => emqx_types:qos() }, - local := #{ - topic => template(), - qos => template() | emqx_types:qos(), - retain => template() | boolean(), - payload => template() | undefined - }, + local := msgvars(), on_message_received := {module(), atom(), [term()]} }. @@ -130,12 +125,14 @@ local => #{ topic => emqx_topic:topic() }, - remote := #{ - topic := template(), - qos => template() | emqx_types:qos(), - retain => template() | boolean(), - payload => template() | undefined - } + remote := msgvars() +}. + +-type msgvars() :: #{ + topic => template(), + qos => template() | emqx_types:qos(), + retain => template() | boolean(), + payload => template() | undefined }. -include_lib("emqx/include/logger.hrl"). @@ -246,9 +243,17 @@ mk_client_options(Config, BridgeOpts) -> force_ping => true }. -mk_client_event_handler(Vars, Opts) when Vars /= undefined -> +mk_client_event_handler(Subscriptions = #{}, Opts) -> + OnMessage = maps:get(on_message_received, Subscriptions, undefined), + LocalPublish = + case Subscriptions of + #{local := Local = #{topic := _}} -> + Local; + #{} -> + undefined + end, #{ - publish => {fun ?MODULE:handle_publish/3, [Vars, Opts]}, + publish => {fun ?MODULE:handle_publish/4, [OnMessage, LocalPublish, Opts]}, disconnected => {fun ?MODULE:handle_disconnect/1, []} }; mk_client_event_handler(undefined, _Opts) -> @@ -279,7 +284,7 @@ ping(Pid) -> send_to_remote(Pid, MsgIn, Conf) -> do_send(Pid, export_msg(MsgIn, Conf)). -do_send(Pid, {true, Msg}) -> +do_send(Pid, Msg) when Msg /= undefined -> case emqtt:publish(Pid, Msg) of ok -> ok; @@ -303,20 +308,18 @@ do_send(Pid, {true, Msg}) -> }), {error, Reason} end; -do_send(_Name, false) -> +do_send(_Name, undefined) -> ok. send_to_remote_async(Pid, MsgIn, Callback, Conf) -> do_send_async(Pid, export_msg(MsgIn, Conf), Callback). -do_send_async(Pid, {true, Msg}, Callback) -> +do_send_async(Pid, Msg, Callback) when Msg /= undefined -> ok = emqtt:publish_async(Pid, Msg, _Timeout = infinity, Callback), {ok, Pid}; -do_send_async(_Pid, false, _Callback) -> +do_send_async(_Pid, undefined, _Callback) -> ok. -pre_process_subscriptions(undefined, _, _) -> - undefined; pre_process_subscriptions( #{remote := RC, local := LC} = Conf, BridgeName, @@ -330,8 +333,6 @@ pre_process_subscriptions(Conf, _, _) when is_map(Conf) -> %% have no 'local' field in the config undefined. -pre_process_forwards(undefined) -> - undefined; pre_process_forwards(#{remote := RC} = Conf) when is_map(Conf) -> Conf#{remote => pre_process_in_out_common(RC)}; pre_process_forwards(Conf) when is_map(Conf) -> @@ -375,44 +376,39 @@ downgrade_ingress_qos(2) -> downgrade_ingress_qos(QoS) -> QoS. -export_msg(Msg, #{forwards := Forwards = #{}}) -> - {true, emqx_connector_mqtt_msg:to_remote_msg(Msg, Forwards)}; +export_msg(Msg, #{forwards := #{remote := Remote}}) -> + to_remote_msg(Msg, Remote); export_msg(Msg, #{forwards := undefined}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", message => Msg, reason => "egress is not configured" }), - false. + undefined. %% -handle_publish(#{properties := Props} = MsgIn, Vars, Opts) -> +handle_publish(#{properties := Props} = MsgIn, OnMessage, LocalPublish, Opts) -> Msg = import_msg(MsgIn, Opts), ?SLOG(debug, #{ msg => "publish_local", - message => Msg, - vars => Vars + message => Msg }), - case Vars of - #{on_message_received := {Mod, Func, Args}} -> - _ = erlang:apply(Mod, Func, [Msg | Args]); - _ -> - ok - end, - maybe_publish_local(Msg, Vars, Props). + maybe_on_message_received(Msg, OnMessage), + maybe_publish_local(Msg, LocalPublish, Props). handle_disconnect(_Reason) -> ok. -maybe_publish_local(Msg, Vars, Props) -> - case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of - %% local topic is not set, discard it - undefined -> - ok; - _ -> - emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props)) - end. +maybe_on_message_received(Msg, {Mod, Func, Args}) -> + erlang:apply(Mod, Func, [Msg | Args]); +maybe_on_message_received(_Msg, undefined) -> + ok. + +maybe_publish_local(Msg, Local = #{}, Props) -> + emqx_broker:publish(to_broker_msg(Msg, Local, Props)); +maybe_publish_local(_Msg, undefined, _Props) -> + ok. import_msg( #{ @@ -459,3 +455,84 @@ printable_maps(Headers) -> #{}, Headers ). + +%% Shame that we have to know the callback module here +%% would be great if we can get rid of #mqtt_msg{} record +%% and use #message{} in all places. +-spec to_remote_msg(emqx_types:message() | map(), msgvars()) -> + #mqtt_msg{}. +to_remote_msg(#message{flags = Flags} = Msg, Vars) -> + {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), + to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars); +to_remote_msg( + MapMsg, + #{ + topic := TopicToken, + qos := QoSToken, + retain := RetainToken + } = Remote +) when is_map(MapMsg) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = process_payload(Remote, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + PubProps = maps:get(pub_props, MapMsg, #{}), + #mqtt_msg{ + qos = QoS, + retain = Retain, + topic = Topic, + props = emqx_utils:pub_props_to_packet(PubProps), + payload = Payload + }. + +%% published from remote node over a MQTT connection +to_broker_msg(Msg, Vars, undefined) -> + to_broker_msg(Msg, Vars, #{}); +to_broker_msg( + #{dup := Dup} = MapMsg, + #{ + topic := TopicToken, + qos := QoSToken, + retain := RetainToken + } = Local, + Props +) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = process_payload(Local, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + PubProps = maps:get(pub_props, MapMsg, #{}), + set_headers( + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, + emqx_message:set_flags( + #{dup => Dup, retain => Retain}, + emqx_message:make(bridge, QoS, Topic, Payload) + ) + ). + +process_payload(From, MapMsg) -> + do_process_payload(maps:get(payload, From, undefined), MapMsg). + +do_process_payload(undefined, Msg) -> + emqx_utils_json:encode(Msg); +do_process_payload(Tks, Msg) -> + replace_vars_in_str(Tks, Msg). + +%% Replace a string contains vars to another string in which the placeholders are replace by the +%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: +%% "a: 1". +replace_vars_in_str(Tokens, Data) when is_list(Tokens) -> + emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary}); +replace_vars_in_str(Val, _Data) -> + Val. + +%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result +%% value will be an integer 1. +replace_simple_var(Tokens, Data) when is_list(Tokens) -> + [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), + Var; +replace_simple_var(Val, _Data) -> + Val. + +set_headers(Val, Msg) -> + emqx_message:set_headers(Val, Msg). From 6967f621d8860f6a5e461bd0cb0df6e9ca72a465 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 22 May 2023 17:24:11 +0300 Subject: [PATCH 04/17] fix(mqttconn): unify error interpretation in sync/async modes Also move this logic to the mqtt connector itself, in order to avoid dealing with extra callback layer. --- .../src/emqx_connector_mqtt.erl | 45 +++++++++++-------- .../src/mqtt/emqx_connector_mqtt_worker.erl | 24 +--------- 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index bd4bf6eb1..cc40b1606 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mqtt). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(supervisor). @@ -141,12 +142,8 @@ on_stop(ResourceId, #{}) -> on_query(ResourceId, {send_message, Msg}, #{worker := Pid, config := Config}) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - case emqx_connector_mqtt_worker:send_to_remote(Pid, Msg, Config) of - ok -> - ok; - {error, Reason} -> - classify_error(Reason) - end. + Result = emqx_connector_mqtt_worker:send_to_remote(Pid, Msg, Config), + handle_send_result(Result). on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{worker := Pid, config := Config}) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), @@ -156,16 +153,12 @@ on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{worker := Pid, con ok; {ok, Pid} -> {ok, Pid}; - {error, Reason} -> - classify_error(Reason) + {error, _} = Error -> + handle_send_result(Error) end. -on_async_result(Callback, ok) -> - apply_callback_function(Callback, ok); -on_async_result(Callback, {ok, _} = Ok) -> - apply_callback_function(Callback, Ok); -on_async_result(Callback, {error, Reason}) -> - apply_callback_function(Callback, classify_error(Reason)). +on_async_result(Callback, Result) -> + apply_callback_function(Callback, handle_send_result(Result)). apply_callback_function(F, Result) when is_function(F) -> erlang:apply(F, [Result]); @@ -177,16 +170,30 @@ apply_callback_function({M, F, A}, Result) when is_atom(M), is_atom(F), is_list( on_get_status(_ResourceId, #{worker := Pid}) -> emqx_connector_mqtt_worker:status(Pid). +handle_send_result(ok) -> + ok; +handle_send_result({ok, #{reason_code := ?RC_SUCCESS}}) -> + ok; +handle_send_result({ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}}) -> + ok; +handle_send_result({ok, Reply}) -> + {error, classify_reply(Reply)}; +handle_send_result({error, Reason}) -> + {error, classify_error(Reason)}. + +classify_reply(Reply = #{reason_code := _}) -> + {unrecoverable_error, Reply}. + classify_error(disconnected = Reason) -> - {error, {recoverable_error, Reason}}; + {recoverable_error, Reason}; classify_error({disconnected, _RC, _} = Reason) -> - {error, {recoverable_error, Reason}}; + {recoverable_error, Reason}; classify_error({shutdown, _} = Reason) -> - {error, {recoverable_error, Reason}}; + {recoverable_error, Reason}; classify_error(shutdown = Reason) -> - {error, {recoverable_error, Reason}}; + {recoverable_error, Reason}; classify_error(Reason) -> - {error, {unrecoverable_error, Reason}}. + {unrecoverable_error, Reason}. make_sub_confs(Subscriptions, _Conf, _) when map_size(Subscriptions) == 0 -> Subscriptions; diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 4169c7f69..8e3ca3136 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -285,29 +285,7 @@ send_to_remote(Pid, MsgIn, Conf) -> do_send(Pid, export_msg(MsgIn, Conf)). do_send(Pid, Msg) when Msg /= undefined -> - case emqtt:publish(Pid, Msg) of - ok -> - ok; - {ok, #{reason_code := RC}} when - RC =:= ?RC_SUCCESS; - RC =:= ?RC_NO_MATCHING_SUBSCRIBERS - -> - ok; - {ok, #{reason_code := RC, reason_code_name := Reason}} -> - ?SLOG(warning, #{ - msg => "remote_publish_failed", - message => Msg, - reason_code => RC, - reason_code_name => Reason - }), - {error, Reason}; - {error, Reason} -> - ?SLOG(info, #{ - msg => "client_failed", - reason => Reason - }), - {error, Reason} - end; + emqtt:publish(Pid, Msg); do_send(_Name, undefined) -> ok. From 81e78516aa907f14853722a0256af785ce0baff6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 23 May 2023 18:54:31 +0300 Subject: [PATCH 05/17] feat(mqttconn): employ ecpool instead of a single worker --- .../test/emqx_bridge_mqtt_SUITE.erl | 10 +- .../src/emqx_connector_mqtt.erl | 156 +++++++----------- .../emqx_connector/src/emqx_connector_sup.erl | 1 - .../src/mqtt/emqx_connector_mqtt_worker.erl | 121 ++++++-------- 4 files changed, 120 insertions(+), 168 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index c00eb6b14..67fb5d019 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -256,11 +256,15 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> } ), - {ok, _, #{state := #{worker := WorkerPid}}} = - emqx_resource:get_instance(emqx_bridge_resource:resource_id(BridgeID)), + ResourceID = emqx_bridge_resource:resource_id(BridgeID), + ClientInfo = ecpool:pick_and_do( + ResourceID, + {emqx_connector_mqtt_worker, info, []}, + no_handover + ), ?assertMatch( #{clean_start := true}, - maps:from_list(emqx_connector_mqtt_worker:info(WorkerPid)) + maps:from_list(ClientInfo) ), %% delete the bridge diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index cc40b1606..30791afe3 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -18,23 +18,13 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --behaviour(supervisor). -behaviour(emqx_resource). -%% API and callbacks for supervisor --export([ - callback_mode/0, - start_link/0, - init/1, - create_bridge/2, - remove_bridge/1, - bridges/0 -]). - -export([on_message_received/3]). %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -44,46 +34,7 @@ -export([on_async_result/2]). -%% =================================================================== -%% supervisor APIs -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - SupFlag = #{ - strategy => one_for_one, - intensity => 100, - period => 10 - }, - {ok, {SupFlag, []}}. - -bridge_spec(Name, Options) -> - #{ - id => Name, - start => {emqx_connector_mqtt_worker, start_link, [Name, Options]}, - restart => temporary, - shutdown => 1000 - }. - --spec bridges() -> [{_Name, _Status}]. -bridges() -> - [ - {Name, emqx_connector_mqtt_worker:status(Name)} - || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE) - ]. - -create_bridge(Name, Options) -> - supervisor:start_child(?MODULE, bridge_spec(Name, Options)). - -remove_bridge(Name) -> - case supervisor:terminate_child(?MODULE, Name) of - ok -> - supervisor:delete_child(?MODULE, Name); - {error, not_found} -> - ok; - {error, Error} -> - {error, Error} - end. +-define(HEALTH_CHECK_TIMEOUT, 1000). %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called @@ -101,24 +52,16 @@ on_start(ResourceId, Conf) -> connector => ResourceId, config => emqx_utils:redact(Conf) }), - BasicConf = basic_config(Conf), - BridgeOpts = BasicConf#{ - clientid => clientid(ResourceId, Conf), + BasicOpts = mk_worker_opts(ResourceId, Conf), + BridgeOpts = BasicOpts#{ subscriptions => make_sub_confs(maps:get(ingress, Conf, #{}), Conf, ResourceId), forwards => maps:get(egress, Conf, #{}) }, - case create_bridge(ResourceId, BridgeOpts) of - {ok, Pid, {ConnProps, WorkerConf}} -> - {ok, #{ - name => ResourceId, - worker => Pid, - config => WorkerConf, - props => ConnProps - }}; - {error, {already_started, _Pid}} -> - ok = remove_bridge(ResourceId), - on_start(ResourceId, Conf); - {error, Reason} -> + {ok, ClientOpts, WorkerConf} = emqx_connector_mqtt_worker:init(ResourceId, BridgeOpts), + case emqx_resource_pool:start(ResourceId, emqx_connector_mqtt_worker, ClientOpts) of + ok -> + {ok, #{config => WorkerConf}}; + {error, {start_pool_failed, _, Reason}} -> {error, Reason} end. @@ -127,34 +70,34 @@ on_stop(ResourceId, #{}) -> msg => "stopping_mqtt_connector", connector => ResourceId }), - case remove_bridge(ResourceId) of - ok -> - ok; - {error, not_found} -> - ok; - {error, Reason} -> - ?SLOG(error, #{ - msg => "stop_mqtt_connector_error", - connector => ResourceId, - reason => Reason - }) - end. + emqx_resource_pool:stop(ResourceId). -on_query(ResourceId, {send_message, Msg}, #{worker := Pid, config := Config}) -> +on_query(ResourceId, {send_message, Msg}, #{config := Config}) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - Result = emqx_connector_mqtt_worker:send_to_remote(Pid, Msg, Config), - handle_send_result(Result). + handle_send_result(with_worker(ResourceId, send_to_remote, [Msg, Config])). -on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{worker := Pid, config := Config}) -> +on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{config := Config}) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - case emqx_connector_mqtt_worker:send_to_remote_async(Pid, Msg, Callback, Config) of + Result = with_worker(ResourceId, send_to_remote_async, [Msg, Callback, Config]), + case Result of ok -> ok; - {ok, Pid} -> + {ok, Pid} when is_pid(Pid) -> {ok, Pid}; - {error, _} = Error -> - handle_send_result(Error) + {error, Reason} -> + {error, classify_error(Reason)} + end. + +with_worker(ResourceId, Fun, Args) -> + Worker = ecpool:get_client(ResourceId), + case is_pid(Worker) andalso ecpool_worker:client(Worker) of + {ok, Client} -> + erlang:apply(emqx_connector_mqtt_worker, Fun, [Client | Args]); + {error, Reason} -> + {error, Reason}; + false -> + {error, disconnected} end. on_async_result(Callback, Result) -> @@ -167,9 +110,6 @@ apply_callback_function({F, A}, Result) when is_function(F), is_list(A) -> apply_callback_function({M, F, A}, Result) when is_atom(M), is_atom(F), is_list(A) -> erlang:apply(M, F, A ++ [Result]). -on_get_status(_ResourceId, #{worker := Pid}) -> - emqx_connector_mqtt_worker:status(Pid). - handle_send_result(ok) -> ok; handle_send_result({ok, #{reason_code := ?RC_SUCCESS}}) -> @@ -195,6 +135,36 @@ classify_error(shutdown = Reason) -> classify_error(Reason) -> {unrecoverable_error, Reason}. +on_get_status(ResourceId, #{}) -> + Workers = [Worker || {_Name, Worker} <- ecpool:workers(ResourceId)], + try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of + Statuses -> + combine_status(Statuses) + catch + exit:timeout -> + connecting + end. + +get_status(Worker) -> + case ecpool_worker:client(Worker) of + {ok, Client} -> + emqx_connector_mqtt_worker:status(Client); + {error, _} -> + disconnected + end. + +combine_status(Statuses) -> + %% NOTE + %% Natural order of statuses: [connected, connecting, disconnected] + %% * `disconnected` wins over any other status + %% * `connecting` wins over `connected` + case lists:reverse(lists:usort(Statuses)) of + [Status | _] -> + Status; + [] -> + disconnected + end. + make_sub_confs(Subscriptions, _Conf, _) when map_size(Subscriptions) == 0 -> Subscriptions; make_sub_confs(Subscriptions, #{hookpoint := HookPoint}, ResourceId) -> @@ -203,7 +173,8 @@ make_sub_confs(Subscriptions, #{hookpoint := HookPoint}, ResourceId) -> make_sub_confs(_SubRemoteConf, Conf, ResourceId) -> error({no_hookpoint_provided, ResourceId, Conf}). -basic_config( +mk_worker_opts( + ResourceId, #{ server := Server, proto_ver := ProtoVer, @@ -215,7 +186,7 @@ basic_config( ssl := #{enable := EnableSsl} = Ssl } = Conf ) -> - BasicConf = #{ + Options = #{ server => Server, %% 30s connect_timeout => 30, @@ -224,6 +195,7 @@ basic_config( %% A load balancing server (such as haproxy) is often set up before the emqx broker server. %% When the load balancing server enables mqtt connection packet inspection, %% non-standard mqtt connection packets might be filtered out by LB. + clientid => clientid(ResourceId, Conf), bridge_mode => BridgeMode, keepalive => ms_to_s(KeepAlive), clean_start => CleanStart, @@ -233,7 +205,7 @@ basic_config( ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }, maps:merge( - BasicConf, + Options, maps:with([username, password], Conf) ). diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 13516813f..21c0f2677 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -33,7 +33,6 @@ init([]) -> period => 20 }, ChildSpecs = [ - child_spec(emqx_connector_mqtt), child_spec(emqx_connector_jwt_sup) ], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 8e3ca3136..7e33a55ca 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -14,51 +14,6 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc Bridge works in two layers (1) batching layer (2) transport layer -%% The `bridge' batching layer collects local messages in batches and sends over -%% to remote MQTT node/cluster via `connection' transport layer. -%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be -%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection' -%% has to be `emqx_connector_mqtt_mod'. -%% -%% ``` -%% +------+ +--------+ -%% | EMQX | | REMOTE | -%% | | | | -%% | (bridge) <==(connection)==> | | -%% | | | | -%% | | | | -%% +------+ +--------+ -%% ''' -%% -%% -%% This module implements 2 kinds of APIs with regards to batching and -%% messaging protocol. (1) A `gen_statem' based local batch collector; -%% (2) APIs for incoming remote batches/messages. -%% -%% Batch collector state diagram -%% -%% [idle] --(0) --> [connecting] --(2)--> [connected] -%% | ^ | -%% | | | -%% '--(1)---'--------(3)------' -%% -%% (0): auto or manual start -%% (1): retry timeout -%% (2): successfully connected to remote node/cluster -%% (3): received {disconnected, Reason} OR -%% failed to send to remote node/cluster. -%% -%% NOTE: A bridge worker may subscribe to multiple (including wildcard) -%% local topics, and the underlying `emqx_bridge_connect' may subscribe to -%% multiple remote topics, however, worker/connections are not designed -%% to support automatic load-balancing, i.e. in case it can not keep up -%% with the amount of messages coming in, administrator should split and -%% balance topics between worker/connections manually. -%% -%% NOTES: -%% * Local messages are all normalised to QoS-1 when exporting to remote - -module(emqx_connector_mqtt_worker). -include_lib("emqx/include/logger.hrl"). @@ -66,7 +21,8 @@ %% APIs -export([ - start_link/2, + init/2, + connect/1, stop/1 ]). @@ -99,6 +55,7 @@ max_inflight := pos_integer(), connect_timeout := pos_integer(), retry_interval := timeout(), + keepalive := non_neg_integer(), bridge_mode := boolean(), ssl := boolean(), ssl_opts := proplists:proplist(), @@ -107,6 +64,11 @@ forwards := map() }. +-type client_option() :: + emqtt:option() + | {name, name()} + | {subscriptions, subscriptions() | undefined}. + -type config() :: #{ subscriptions := subscriptions() | undefined, forwards := forwards() | undefined @@ -138,50 +100,64 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -%% @doc Start a bridge worker. Supported configs: -%% mountpoint: The topic mount point for messages sent to remote node/cluster -%% `undefined', `<<>>' or `""' to disable -%% forwards: Local topics to subscribe. -%% -%% Find more connection specific configs in the callback modules -%% of emqx_bridge_connect behaviour. --spec start_link(name(), options()) -> - {ok, pid(), {emqtt:properties(), config()}} | {error, _Reason}. -start_link(Name, BridgeOpts) -> +-spec init(name(), options()) -> + {ok, [client_option()], config()}. +init(Name, BridgeOpts) -> + Config = init_config(Name, BridgeOpts), + ClientOpts0 = mk_client_options(Config, BridgeOpts), + ClientOpts = ClientOpts0#{ + name => Name, + subscriptions => maps:get(subscriptions, Config) + }, + {ok, maps:to_list(ClientOpts), Config}. + +%% @doc Start a bridge worker. +-spec connect([client_option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(ClientOpts0) -> ?SLOG(debug, #{ msg => "client_starting", - name => Name, - options => BridgeOpts + options => emqx_utils:redact(ClientOpts0) }), - Config = init_config(Name, BridgeOpts), - Options = mk_client_options(Config, BridgeOpts), - case emqtt:start_link(Options) of + {value, {_, Name}, ClientOpts1} = lists:keytake(name, 1, ClientOpts0), + {value, {_, WorkerId}, ClientOpts} = lists:keytake(ecpool_worker_id, 1, ClientOpts1), + case emqtt:start_link(mk_emqtt_opts(WorkerId, ClientOpts)) of {ok, Pid} -> - connect(Pid, Name, Config); + connect(Pid, Name, WorkerId, ClientOpts); {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", - config => emqx_utils:redact(BridgeOpts), + config => emqx_utils:redact(ClientOpts), reason => Reason }), Error end. -connect(Pid, Name, Config = #{subscriptions := Subscriptions}) -> +mk_emqtt_opts(WorkerId, ClientOpts) -> + {_, ClientId} = lists:keyfind(clientid, 1, ClientOpts), + lists:keystore(clientid, 1, ClientOpts, {clientid, mk_clientid(WorkerId, ClientId)}). + +mk_clientid(WorkerId, ClientId) -> + iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + +connect(Pid, Name, WorkerId, ClientOpts) -> case emqtt:connect(Pid) of - {ok, Props} -> - case subscribe_remote_topics(Pid, Subscriptions) of - ok -> - {ok, Pid, {Props, Config}}; + {ok, _Props} -> + % NOTE + % Subscribe to remote topics only when the first worker is started. + Subscriptions = proplists:get_value(subscriptions, ClientOpts), + case WorkerId =:= 1 andalso subscribe_remote_topics(Pid, Subscriptions) of + false -> + {ok, Pid}; {ok, _, _RCs} -> - {ok, Pid, {Props, Config}}; + {ok, Pid}; {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_subscribe_failed", subscriptions => Subscriptions, reason => Reason }), - _ = emqtt:stop(Pid), + _ = catch emqtt:stop(Pid), Error end; {error, Reason} = Error -> @@ -190,14 +166,14 @@ connect(Pid, Name, Config = #{subscriptions := Subscriptions}) -> reason => Reason, name => Name }), - _ = emqtt:stop(Pid), + _ = catch emqtt:stop(Pid), Error end. subscribe_remote_topics(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> emqtt:subscribe(Pid, RemoteTopic, QoS); subscribe_remote_topics(_Ref, undefined) -> - ok. + false. init_config(Name, Opts) -> Subscriptions = maps:get(subscriptions, Opts, undefined), @@ -230,6 +206,7 @@ mk_client_options(Config, BridgeOpts) -> max_inflight, connect_timeout, retry_interval, + keepalive, bridge_mode, ssl, ssl_opts From 4e6269bedb6beca58c280f21f847c7be9f0d5be5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 13:57:55 +0300 Subject: [PATCH 06/17] feat(mqttconn): subscribe each worker if shared subcription Also rename `subscriptions` -> `ingress` and `forwards` -> `egress` for consistency with the config schema. --- .../test/emqx_bridge_mqtt_SUITE.erl | 48 ++++++++++ .../src/emqx_connector_mqtt.erl | 14 +-- .../src/mqtt/emqx_connector_mqtt_worker.erl | 92 +++++++++---------- 3 files changed, 98 insertions(+), 56 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index 67fb5d019..aecb04e03 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -221,6 +221,12 @@ t_mqtt_conn_bridge_ingress(_) -> request(put, uri(["bridges", BridgeIDIngress]), ServerConf) ), + %% non-shared subscription, verify that only one client is subscribed + ?assertEqual( + 1, + length(emqx:subscribers(<>)) + ), + %% we now test if the bridge works as expected RemoteTopic = <>, LocalTopic = <>, @@ -245,6 +251,48 @@ t_mqtt_conn_bridge_ingress(_) -> ok. +t_mqtt_conn_bridge_ingress_shared_subscription(_) -> + PoolSize = 4, + Ns = lists:seq(1, 10), + BridgeName = atom_to_binary(?FUNCTION_NAME), + BridgeID = create_bridge( + ?SERVER_CONF(<<>>)#{ + <<"type">> => ?TYPE_MQTT, + <<"name">> => BridgeName, + <<"pool_size">> => PoolSize, + <<"ingress">> => #{ + <<"remote">> => #{ + <<"topic">> => <<"$share/ingress/", ?INGRESS_REMOTE_TOPIC, "/#">>, + <<"qos">> => 1 + }, + <<"local">> => #{ + <<"topic">> => <>, + <<"qos">> => <<"${qos}">>, + <<"payload">> => <<"${clientid}">>, + <<"retain">> => <<"${retain}">> + } + } + } + ), + + RemoteTopic = <>, + LocalTopic = <>, + ok = emqx:subscribe(LocalTopic), + + _ = emqx_utils:pmap( + fun emqx:publish/1, + [emqx_message:make(RemoteTopic, <<>>) || _ <- Ns] + ), + _ = [assert_mqtt_msg_received(LocalTopic) || _ <- Ns], + + ?assertEqual( + PoolSize, + length(emqx_shared_sub:subscribers(<<"ingress">>, <>)) + ), + + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + ok. + t_mqtt_egress_bridge_ignores_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 30791afe3..09228254a 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -54,8 +54,8 @@ on_start(ResourceId, Conf) -> }), BasicOpts = mk_worker_opts(ResourceId, Conf), BridgeOpts = BasicOpts#{ - subscriptions => make_sub_confs(maps:get(ingress, Conf, #{}), Conf, ResourceId), - forwards => maps:get(egress, Conf, #{}) + ingress => mk_ingress_config(maps:get(ingress, Conf, #{}), Conf, ResourceId), + egress => maps:get(egress, Conf, #{}) }, {ok, ClientOpts, WorkerConf} = emqx_connector_mqtt_worker:init(ResourceId, BridgeOpts), case emqx_resource_pool:start(ResourceId, emqx_connector_mqtt_worker, ClientOpts) of @@ -165,12 +165,12 @@ combine_status(Statuses) -> disconnected end. -make_sub_confs(Subscriptions, _Conf, _) when map_size(Subscriptions) == 0 -> - Subscriptions; -make_sub_confs(Subscriptions, #{hookpoint := HookPoint}, ResourceId) -> +mk_ingress_config(Ingress, _Conf, _) when map_size(Ingress) == 0 -> + Ingress; +mk_ingress_config(Ingress, #{hookpoint := HookPoint}, ResourceId) -> MFA = {?MODULE, on_message_received, [HookPoint, ResourceId]}, - Subscriptions#{on_message_received => MFA}; -make_sub_confs(_SubRemoteConf, Conf, ResourceId) -> + Ingress#{on_message_received => MFA}; +mk_ingress_config(_Ingress, Conf, ResourceId) -> error({no_hookpoint_provided, ResourceId, Conf}). mk_worker_opts( diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 7e33a55ca..ede477602 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -60,21 +60,18 @@ ssl := boolean(), ssl_opts := proplists:proplist(), % bridge options - subscriptions := map(), - forwards := map() + ingress := map(), + egress := map() }. -type client_option() :: emqtt:option() | {name, name()} - | {subscriptions, subscriptions() | undefined}. + | {ingress, ingress() | undefined}. --type config() :: #{ - subscriptions := subscriptions() | undefined, - forwards := forwards() | undefined -}. +-type config() :: egress() | undefined. --type subscriptions() :: #{ +-type ingress() :: #{ remote := #{ topic := emqx_topic:topic(), qos => emqx_types:qos() @@ -83,7 +80,7 @@ on_message_received := {module(), atom(), [term()]} }. --type forwards() :: #{ +-type egress() :: #{ local => #{ topic => emqx_topic:topic() }, @@ -103,13 +100,10 @@ -spec init(name(), options()) -> {ok, [client_option()], config()}. init(Name, BridgeOpts) -> - Config = init_config(Name, BridgeOpts), - ClientOpts0 = mk_client_options(Config, BridgeOpts), - ClientOpts = ClientOpts0#{ - name => Name, - subscriptions => maps:get(subscriptions, Config) - }, - {ok, maps:to_list(ClientOpts), Config}. + Ingress = pre_process_ingress(maps:get(ingress, BridgeOpts), Name, BridgeOpts), + Egress = pre_process_egress(maps:get(egress, BridgeOpts)), + ClientOpts = mk_client_options(Name, Ingress, BridgeOpts), + {ok, maps:to_list(ClientOpts), Egress}. %% @doc Start a bridge worker. -spec connect([client_option() | {ecpool_worker_id, pos_integer()}]) -> @@ -134,7 +128,7 @@ connect(ClientOpts0) -> end. mk_emqtt_opts(WorkerId, ClientOpts) -> - {_, ClientId} = lists:keyfind(clientid, 1, ClientOpts), + ClientId = proplists:get_value(clientid, ClientOpts), lists:keystore(clientid, 1, ClientOpts, {clientid, mk_clientid(WorkerId, ClientId)}). mk_clientid(WorkerId, ClientId) -> @@ -143,10 +137,8 @@ mk_clientid(WorkerId, ClientId) -> connect(Pid, Name, WorkerId, ClientOpts) -> case emqtt:connect(Pid) of {ok, _Props} -> - % NOTE - % Subscribe to remote topics only when the first worker is started. - Subscriptions = proplists:get_value(subscriptions, ClientOpts), - case WorkerId =:= 1 andalso subscribe_remote_topics(Pid, Subscriptions) of + Ingress = proplists:get_value(ingress, ClientOpts), + case subscribe_remote_topic(Pid, WorkerId, Ingress) of false -> {ok, Pid}; {ok, _, _RCs} -> @@ -154,7 +146,7 @@ connect(Pid, Name, WorkerId, ClientOpts) -> {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_subscribe_failed", - subscriptions => Subscriptions, + ingress => Ingress, reason => Reason }), _ = catch emqtt:stop(Pid), @@ -170,25 +162,25 @@ connect(Pid, Name, WorkerId, ClientOpts) -> Error end. -subscribe_remote_topics(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> - emqtt:subscribe(Pid, RemoteTopic, QoS); -subscribe_remote_topics(_Ref, undefined) -> +subscribe_remote_topic(Pid, WorkerId, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> + case emqx_topic:parse(RemoteTopic) of + {_Filter, #{share := _Name}} -> + % NOTE: this is shared subscription, each worker may subscribe + emqtt:subscribe(Pid, RemoteTopic, QoS); + {_Filter, #{}} when WorkerId =:= 1 -> + % NOTE: this is regular subscription, only the first worker should subscribe + emqtt:subscribe(Pid, RemoteTopic, QoS); + {_Filter, #{}} -> + false + end; +subscribe_remote_topic(_Ref, _, undefined) -> false. -init_config(Name, Opts) -> - Subscriptions = maps:get(subscriptions, Opts, undefined), - Forwards = maps:get(forwards, Opts, undefined), - #{ - subscriptions => pre_process_subscriptions(Subscriptions, Name, Opts), - forwards => pre_process_forwards(Forwards) - }. - -mk_client_options(Config, BridgeOpts) -> +mk_client_options(Name, Ingress, BridgeOpts) -> Server = iolist_to_binary(maps:get(server, BridgeOpts)), HostPort = emqx_connector_mqtt_schema:parse_server(Server), - Subscriptions = maps:get(subscriptions, Config), CleanStart = - case Subscriptions of + case Ingress of #{remote := _} -> maps:get(clean_start, BridgeOpts); undefined -> @@ -214,16 +206,18 @@ mk_client_options(Config, BridgeOpts) -> BridgeOpts ), Opts#{ - msg_handler => mk_client_event_handler(Subscriptions, #{server => Server}), + name => Name, + ingress => Ingress, + msg_handler => mk_client_event_handler(Ingress, #{server => Server}), hosts => [HostPort], clean_start => CleanStart, force_ping => true }. -mk_client_event_handler(Subscriptions = #{}, Opts) -> - OnMessage = maps:get(on_message_received, Subscriptions, undefined), +mk_client_event_handler(Ingress = #{}, Opts) -> + OnMessage = maps:get(on_message_received, Ingress, undefined), LocalPublish = - case Subscriptions of + case Ingress of #{local := Local = #{topic := _}} -> Local; #{} -> @@ -275,26 +269,26 @@ do_send_async(Pid, Msg, Callback) when Msg /= undefined -> do_send_async(_Pid, undefined, _Callback) -> ok. -pre_process_subscriptions( +pre_process_ingress( #{remote := RC, local := LC} = Conf, BridgeName, BridgeOpts ) when is_map(Conf) -> Conf#{ remote => pre_process_in_remote(RC, BridgeName, BridgeOpts), - local => pre_process_in_out_common(LC) + local => pre_process_common(LC) }; -pre_process_subscriptions(Conf, _, _) when is_map(Conf) -> +pre_process_ingress(Conf, _, _) when is_map(Conf) -> %% have no 'local' field in the config undefined. -pre_process_forwards(#{remote := RC} = Conf) when is_map(Conf) -> - Conf#{remote => pre_process_in_out_common(RC)}; -pre_process_forwards(Conf) when is_map(Conf) -> +pre_process_egress(#{remote := RC} = Conf) when is_map(Conf) -> + Conf#{remote => pre_process_common(RC)}; +pre_process_egress(Conf) when is_map(Conf) -> %% have no 'remote' field in the config undefined. -pre_process_in_out_common(Conf0) -> +pre_process_common(Conf0) -> Conf1 = pre_process_conf(topic, Conf0), Conf2 = pre_process_conf(qos, Conf1), Conf3 = pre_process_conf(payload, Conf2), @@ -331,9 +325,9 @@ downgrade_ingress_qos(2) -> downgrade_ingress_qos(QoS) -> QoS. -export_msg(Msg, #{forwards := #{remote := Remote}}) -> +export_msg(Msg, #{remote := Remote}) -> to_remote_msg(Msg, Remote); -export_msg(Msg, #{forwards := undefined}) -> +export_msg(Msg, undefined) -> ?SLOG(error, #{ msg => "forwarding_unavailable", message => Msg, From 6e97dffdb8a0b9941681e0bf599eaa574665c4d1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 14:01:59 +0300 Subject: [PATCH 07/17] feat(mqttconn): deprecate `mode` config parameter It adds no value: the only mode was `cluster_shareload` and we just as well can decide to "share" the load across cluster just by looking if the remote topic is shared subcription filter or not. --- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 2a40980af..a30de141e 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -68,7 +68,8 @@ fields("server_configs") -> hoconsc:enum([cluster_shareload]), #{ default => cluster_shareload, - desc => ?DESC("mode") + desc => ?DESC("mode"), + deprecated => {since, "v5.1.0 & e5.1.0"} } )}, {server, emqx_schema:servers_sc(#{desc => ?DESC("server")}, ?MQTT_HOST_OPTS)}, From c7528e9b35c0f55a76bc1c008d5c14d8a3d0f44d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 14:06:25 +0300 Subject: [PATCH 08/17] feat(mqttconn): add `pool_size` config parameter That currently tunes the number of MQTT clients employed both for subscriptions (if shared subscription is used) and for publishing to a remote broker. --- apps/emqx_connector/src/emqx_connector_mqtt.erl | 2 ++ apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 1 + apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl | 3 +++ 3 files changed, 6 insertions(+) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 09228254a..0658c28a7 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -177,6 +177,7 @@ mk_worker_opts( ResourceId, #{ server := Server, + pool_size := PoolSize, proto_ver := ProtoVer, bridge_mode := BridgeMode, clean_start := CleanStart, @@ -188,6 +189,7 @@ mk_worker_opts( ) -> Options = #{ server => Server, + pool_size => PoolSize, %% 30s connect_timeout => 30, proto_ver => ProtoVer, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index a30de141e..f02fd19ad 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -73,6 +73,7 @@ fields("server_configs") -> } )}, {server, emqx_schema:servers_sc(#{desc => ?DESC("server")}, ?MQTT_HOST_OPTS)}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {clientid_prefix, mk(binary(), #{required => false, desc => ?DESC("clientid_prefix")})}, {reconnect_interval, mk(string(), #{deprecated => {since, "v5.0.16"}})}, {proto_ver, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index ede477602..223d4d058 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -46,6 +46,7 @@ -type options() :: #{ % endpoint server := iodata(), + pool_size := pos_integer(), % emqtt client options proto_ver := v3 | v4 | v5, username := binary(), @@ -66,6 +67,7 @@ -type client_option() :: emqtt:option() + | {pool_size, pos_integer()} | {name, name()} | {ingress, ingress() | undefined}. @@ -191,6 +193,7 @@ mk_client_options(Name, Ingress, BridgeOpts) -> end, Opts = maps:with( [ + pool_size, proto_ver, username, password, From a5fc26736deed3675c6614c56f1f44e891b240fc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 19:43:20 +0300 Subject: [PATCH 09/17] refactor(mqttconn): split ingress/egress into 2 separate pools Each with a more refined set of responsibilities, at the cost of slight code duplication. Also provide two different config fields for each pool size. --- .../test/emqx_bridge_api_SUITE.erl | 9 +- .../test/emqx_bridge_mqtt_SUITE.erl | 10 +- .../src/emqx_connector_mqtt.erl | 244 ++++++--- .../src/mqtt/emqx_connector_mqtt_egress.erl | 162 ++++++ .../src/mqtt/emqx_connector_mqtt_ingress.erl | 272 ++++++++++ .../src/mqtt/emqx_connector_mqtt_msg.erl | 95 ++++ .../src/mqtt/emqx_connector_mqtt_schema.erl | 21 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 490 ------------------ apps/emqx_resource/src/emqx_resource.erl | 3 +- rel/i18n/emqx_connector_mqtt_schema.hocon | 18 + 10 files changed, 761 insertions(+), 563 deletions(-) create mode 100644 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl create mode 100644 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl create mode 100644 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl delete mode 100644 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 1ac6750a4..ecab986e8 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -47,7 +47,14 @@ <<"server">> => SERVER, <<"username">> => <<"user1">>, <<"password">> => <<"">>, - <<"proto_ver">> => <<"v5">> + <<"proto_ver">> => <<"v5">>, + <<"egress">> => #{ + <<"remote">> => #{ + <<"topic">> => <<"emqx/${topic}">>, + <<"qos">> => <<"${qos}">>, + <<"retain">> => false + } + } }). -define(MQTT_BRIDGE(SERVER), ?MQTT_BRIDGE(SERVER, <<"mqtt_egress_test_bridge">>)). diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index aecb04e03..6c36e08e7 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -22,9 +22,7 @@ -include("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --include("emqx_dashboard/include/emqx_dashboard.hrl"). %% output functions -export([inspect/3]). @@ -259,8 +257,8 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> ?SERVER_CONF(<<>>)#{ <<"type">> => ?TYPE_MQTT, <<"name">> => BridgeName, - <<"pool_size">> => PoolSize, <<"ingress">> => #{ + <<"pool_size">> => PoolSize, <<"remote">> => #{ <<"topic">> => <<"$share/ingress/", ?INGRESS_REMOTE_TOPIC, "/#">>, <<"qos">> => 1 @@ -305,9 +303,11 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> ), ResourceID = emqx_bridge_resource:resource_id(BridgeID), + {ok, _Group, #{state := #{egress_pool_name := EgressPoolName}}} = + emqx_resource_manager:lookup_cached(ResourceID), ClientInfo = ecpool:pick_and_do( - ResourceID, - {emqx_connector_mqtt_worker, info, []}, + EgressPoolName, + {emqx_connector_mqtt_egress, info, []}, no_handover ), ?assertMatch( diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 0658c28a7..a75f4db39 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -52,34 +52,134 @@ on_start(ResourceId, Conf) -> connector => ResourceId, config => emqx_utils:redact(Conf) }), - BasicOpts = mk_worker_opts(ResourceId, Conf), - BridgeOpts = BasicOpts#{ - ingress => mk_ingress_config(maps:get(ingress, Conf, #{}), Conf, ResourceId), - egress => maps:get(egress, Conf, #{}) - }, - {ok, ClientOpts, WorkerConf} = emqx_connector_mqtt_worker:init(ResourceId, BridgeOpts), - case emqx_resource_pool:start(ResourceId, emqx_connector_mqtt_worker, ClientOpts) of + case start_ingress(ResourceId, Conf) of + {ok, Result1} -> + case start_egress(ResourceId, Conf) of + {ok, Result2} -> + {ok, maps:merge(Result1, Result2)}; + {error, Reason} -> + _ = stop_ingress(Result1), + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +start_ingress(ResourceId, Conf) -> + ClientOpts = mk_client_opts(ResourceId, "ingress", Conf), + case mk_ingress_config(ResourceId, Conf) of + Ingress = #{} -> + start_ingress(ResourceId, Ingress, ClientOpts); + undefined -> + {ok, #{}} + end. + +start_ingress(ResourceId, Ingress, ClientOpts) -> + PoolName = <>, + PoolSize = choose_ingress_pool_size(Ingress), + Options = [ + {name, PoolName}, + {pool_size, PoolSize}, + {ingress, Ingress}, + {client_opts, ClientOpts} + ], + case emqx_resource_pool:start(PoolName, emqx_connector_mqtt_ingress, Options) of ok -> - {ok, #{config => WorkerConf}}; + {ok, #{ingress_pool_name => PoolName}}; {error, {start_pool_failed, _, Reason}} -> {error, Reason} end. -on_stop(ResourceId, #{}) -> +choose_ingress_pool_size(#{remote := #{topic := RemoteTopic}, pool_size := PoolSize}) -> + case emqx_topic:parse(RemoteTopic) of + {_Filter, #{share := _Name}} -> + % NOTE: this is shared subscription, many workers may subscribe + PoolSize; + {_Filter, #{}} -> + % NOTE: this is regular subscription, only one worker should subscribe + ?SLOG(warning, #{ + msg => "ingress_pool_size_ignored", + reason => + "Remote topic filter is not a shared subscription, " + "ingress pool will start with a single worker", + config_pool_size => PoolSize, + pool_size => 1 + }), + 1 + end. + +start_egress(ResourceId, Conf) -> + % NOTE + % We are ignoring the user configuration here because there's currently no reliable way + % to ensure proper session recovery according to the MQTT spec. + ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, "egress", Conf)), + case mk_egress_config(Conf) of + Egress = #{} -> + start_egress(ResourceId, Egress, ClientOpts); + undefined -> + {ok, #{}} + end. + +start_egress(ResourceId, Egress, ClientOpts) -> + PoolName = <>, + PoolSize = maps:get(pool_size, Egress), + Options = [ + {name, PoolName}, + {pool_size, PoolSize}, + {client_opts, ClientOpts} + ], + case emqx_resource_pool:start(PoolName, emqx_connector_mqtt_egress, Options) of + ok -> + {ok, #{ + egress_pool_name => PoolName, + egress_config => emqx_connector_mqtt_egress:config(Egress) + }}; + {error, {start_pool_failed, _, Reason}} -> + {error, Reason} + end. + +on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", connector => ResourceId }), - emqx_resource_pool:stop(ResourceId). + ok = stop_ingress(State), + ok = stop_egress(State). -on_query(ResourceId, {send_message, Msg}, #{config := Config}) -> +stop_ingress(#{ingress_pool_name := PoolName}) -> + emqx_resource_pool:stop(PoolName); +stop_ingress(#{}) -> + ok. + +stop_egress(#{egress_pool_name := PoolName}) -> + emqx_resource_pool:stop(PoolName); +stop_egress(#{}) -> + ok. + +on_query( + ResourceId, + {send_message, Msg}, + #{egress_pool_name := PoolName, egress_config := Config} +) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - handle_send_result(with_worker(ResourceId, send_to_remote, [Msg, Config])). + handle_send_result(with_worker(PoolName, send, [Msg, Config])); +on_query(ResourceId, {send_message, Msg}, #{}) -> + ?SLOG(error, #{ + msg => "forwarding_unavailable", + connector => ResourceId, + message => Msg, + reason => "Egress is not configured" + }). -on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{config := Config}) -> +on_query_async( + ResourceId, + {send_message, Msg}, + CallbackIn, + #{egress_pool_name := PoolName, egress_config := Config} +) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - Result = with_worker(ResourceId, send_to_remote_async, [Msg, Callback, Config]), + Result = with_worker(PoolName, send_async, [Msg, Callback, Config]), case Result of ok -> ok; @@ -87,13 +187,20 @@ on_query_async(ResourceId, {send_message, Msg}, CallbackIn, #{config := Config}) {ok, Pid}; {error, Reason} -> {error, classify_error(Reason)} - end. + end; +on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> + ?SLOG(error, #{ + msg => "forwarding_unavailable", + connector => ResourceId, + message => Msg, + reason => "Egress is not configured" + }). with_worker(ResourceId, Fun, Args) -> Worker = ecpool:get_client(ResourceId), case is_pid(Worker) andalso ecpool_worker:client(Worker) of {ok, Client} -> - erlang:apply(emqx_connector_mqtt_worker, Fun, [Client | Args]); + erlang:apply(emqx_connector_mqtt_egress, Fun, [Client | Args]); {error, Reason} -> {error, Reason}; false -> @@ -135,8 +242,9 @@ classify_error(shutdown = Reason) -> classify_error(Reason) -> {unrecoverable_error, Reason}. -on_get_status(ResourceId, #{}) -> - Workers = [Worker || {_Name, Worker} <- ecpool:workers(ResourceId)], +on_get_status(_ResourceId, State) -> + Pools = maps:to_list(maps:with([ingress_pool_name, egress_pool_name], State)), + Workers = [{Pool, Worker} || {Pool, PN} <- Pools, {_Name, Worker} <- ecpool:workers(PN)], try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of Statuses -> combine_status(Statuses) @@ -145,10 +253,12 @@ on_get_status(ResourceId, #{}) -> connecting end. -get_status(Worker) -> +get_status({Pool, Worker}) -> case ecpool_worker:client(Worker) of - {ok, Client} -> - emqx_connector_mqtt_worker:status(Client); + {ok, Client} when Pool == ingress_pool_name -> + emqx_connector_mqtt_ingress:status(Client); + {ok, Client} when Pool == egress_pool_name -> + emqx_connector_mqtt_egress:status(Client); {error, _} -> disconnected end. @@ -165,56 +275,68 @@ combine_status(Statuses) -> disconnected end. -mk_ingress_config(Ingress, _Conf, _) when map_size(Ingress) == 0 -> - Ingress; -mk_ingress_config(Ingress, #{hookpoint := HookPoint}, ResourceId) -> - MFA = {?MODULE, on_message_received, [HookPoint, ResourceId]}, - Ingress#{on_message_received => MFA}; -mk_ingress_config(_Ingress, Conf, ResourceId) -> - error({no_hookpoint_provided, ResourceId, Conf}). - -mk_worker_opts( +mk_ingress_config( ResourceId, #{ + ingress := Ingress = #{remote := _}, server := Server, - pool_size := PoolSize, - proto_ver := ProtoVer, - bridge_mode := BridgeMode, - clean_start := CleanStart, - keepalive := KeepAlive, - retry_interval := RetryIntv, - max_inflight := MaxInflight, - ssl := #{enable := EnableSsl} = Ssl - } = Conf + hookpoint := HookPoint + } ) -> - Options = #{ + Ingress#{ server => Server, - pool_size => PoolSize, - %% 30s + on_message_received => {?MODULE, on_message_received, [HookPoint, ResourceId]} + }; +mk_ingress_config(ResourceId, #{ingress := #{remote := _}} = Conf) -> + error({no_hookpoint_provided, ResourceId, Conf}); +mk_ingress_config(_ResourceId, #{}) -> + undefined. + +mk_egress_config(#{egress := Egress = #{remote := _}}) -> + Egress; +mk_egress_config(#{}) -> + undefined. + +mk_client_opts( + ResourceId, + ClientScope, + Config = #{ + server := Server, + keepalive := KeepAlive, + ssl := #{enable := EnableSsl} = Ssl + } +) -> + HostPort = emqx_connector_mqtt_schema:parse_server(Server), + Options = maps:with( + [ + proto_ver, + username, + password, + clean_start, + retry_interval, + max_inflight, + % Opening a connection in bridge mode will form a non-standard mqtt connection message. + % A load balancing server (such as haproxy) is often set up before the emqx broker server. + % When the load balancing server enables mqtt connection packet inspection, + % non-standard mqtt connection packets might be filtered out by LB. + bridge_mode + ], + Config + ), + Options#{ + hosts => [HostPort], + clientid => clientid(ResourceId, ClientScope, Config), connect_timeout => 30, - proto_ver => ProtoVer, - %% Opening a connection in bridge mode will form a non-standard mqtt connection message. - %% A load balancing server (such as haproxy) is often set up before the emqx broker server. - %% When the load balancing server enables mqtt connection packet inspection, - %% non-standard mqtt connection packets might be filtered out by LB. - clientid => clientid(ResourceId, Conf), - bridge_mode => BridgeMode, keepalive => ms_to_s(KeepAlive), - clean_start => CleanStart, - retry_interval => RetryIntv, - max_inflight => MaxInflight, + force_ping => true, ssl => EnableSsl, ssl_opts => maps:to_list(maps:remove(enable, Ssl)) - }, - maps:merge( - Options, - maps:with([username, password], Conf) - ). + }. ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Id, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) -> - iolist_to_binary([Prefix, ":", Id, ":", atom_to_list(node())]); -clientid(Id, _Conf) -> - iolist_to_binary([Id, ":", atom_to_list(node())]). +clientid(Id, ClientScope, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) -> + iolist_to_binary([Prefix, ":", Id, ":", ClientScope, ":", atom_to_list(node())]); +clientid(Id, ClientScope, _Conf) -> + iolist_to_binary([Id, ":", ClientScope, ":", atom_to_list(node())]). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl new file mode 100644 index 000000000..0e413cbc9 --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl @@ -0,0 +1,162 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_egress). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). + +-behaviour(ecpool_worker). + +%% ecpool +-export([connect/1]). + +-export([ + config/1, + send/3, + send_async/4 +]). + +%% management APIs +-export([ + status/1, + info/1 +]). + +-type name() :: term(). +-type message() :: emqx_types:message() | map(). +-type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. +-type remote_message() :: #mqtt_msg{}. + +-type option() :: + {name, name()} + %% see `emqtt:option()` + | {client_opts, map()}. + +-type egress() :: #{ + local => #{ + topic => emqx_topic:topic() + }, + remote := emqx_connector_mqtt_msg:msgvars() +}. + +%% @doc Start an ingress bridge worker. +-spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(Options) -> + ?SLOG(debug, #{ + msg => "egress_client_starting", + options => emqx_utils:redact(Options) + }), + Name = proplists:get_value(name, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ClientOpts = proplists:get_value(client_opts, Options), + case emqtt:start_link(mk_client_opts(WorkerId, ClientOpts)) of + {ok, Pid} -> + connect(Pid, Name); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "egress_client_start_failed", + config => emqx_utils:redact(ClientOpts), + reason => Reason + }), + Error + end. + +mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) -> + ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}. + +mk_clientid(WorkerId, ClientId) -> + iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + +connect(Pid, Name) -> + case emqtt:connect(Pid) of + {ok, _Props} -> + {ok, Pid}; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "egress_client_connect_failed", + reason => Reason, + name => Name + }), + _ = catch emqtt:stop(Pid), + Error + end. + +%% + +-spec config(map()) -> + egress(). +config(#{remote := RC = #{}} = Conf) -> + Conf#{remote => emqx_connector_mqtt_msg:parse(RC)}. + +-spec send(pid(), message(), egress()) -> + ok. +send(Pid, MsgIn, Egress) -> + emqtt:publish(Pid, export_msg(MsgIn, Egress)). + +-spec send_async(pid(), message(), callback(), egress()) -> + ok | {ok, pid()}. +send_async(Pid, MsgIn, Callback, Egress) -> + ok = emqtt:publish_async(Pid, export_msg(MsgIn, Egress), _Timeout = infinity, Callback), + {ok, Pid}. + +export_msg(Msg, #{remote := Remote}) -> + to_remote_msg(Msg, Remote). + +-spec to_remote_msg(message(), emqx_connector_mqtt_msg:msgvars()) -> + remote_message(). +to_remote_msg(#message{flags = Flags} = Msg, Vars) -> + {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), + to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars); +to_remote_msg(Msg = #{}, Remote) -> + #{ + topic := Topic, + payload := Payload, + qos := QoS, + retain := Retain + } = emqx_connector_mqtt_msg:render(Msg, Remote), + PubProps = maps:get(pub_props, Msg, #{}), + #mqtt_msg{ + qos = QoS, + retain = Retain, + topic = Topic, + props = emqx_utils:pub_props_to_packet(PubProps), + payload = Payload + }. + +%% + +-spec info(pid()) -> + [{atom(), term()}]. +info(Pid) -> + emqtt:info(Pid). + +-spec status(pid()) -> + emqx_resource:resource_status(). +status(Pid) -> + try + case proplists:get_value(socket, info(Pid)) of + Socket when Socket /= undefined -> + connected; + undefined -> + connecting + end + catch + exit:{noproc, _} -> + disconnected + end. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl new file mode 100644 index 000000000..c11895c49 --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl @@ -0,0 +1,272 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_ingress). + +-include_lib("emqx/include/logger.hrl"). + +-behaviour(ecpool_worker). + +%% ecpool +-export([connect/1]). + +%% management APIs +-export([ + status/1, + info/1 +]). + +-export([handle_publish/4]). +-export([handle_disconnect/1]). + +-type name() :: term(). + +-type option() :: + {name, name()} + | {ingress, map()} + %% see `emqtt:option()` + | {client_opts, map()}. + +-type ingress() :: #{ + server := string(), + remote := #{ + topic := emqx_topic:topic(), + qos => emqx_types:qos() + }, + local := emqx_connector_mqtt_msg:msgvars(), + on_message_received := {module(), atom(), [term()]} +}. + +%% @doc Start an ingress bridge worker. +-spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(Options) -> + ?SLOG(debug, #{ + msg => "ingress_client_starting", + options => emqx_utils:redact(Options) + }), + Name = proplists:get_value(name, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + Ingress = config(proplists:get_value(ingress, Options), Name), + ClientOpts = proplists:get_value(client_opts, Options), + case emqtt:start_link(mk_client_opts(WorkerId, Ingress, ClientOpts)) of + {ok, Pid} -> + connect(Pid, Name, Ingress); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "client_start_failed", + config => emqx_utils:redact(ClientOpts), + reason => Reason + }), + Error + end. + +mk_client_opts(WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> + ClientOpts#{ + clientid := mk_clientid(WorkerId, ClientId), + msg_handler => mk_client_event_handler(Ingress) + }. + +mk_clientid(WorkerId, ClientId) -> + iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + +mk_client_event_handler(Ingress = #{}) -> + IngressVars = maps:with([server], Ingress), + OnMessage = maps:get(on_message_received, Ingress, undefined), + LocalPublish = + case Ingress of + #{local := Local = #{topic := _}} -> + Local; + #{} -> + undefined + end, + #{ + publish => {fun ?MODULE:handle_publish/4, [OnMessage, LocalPublish, IngressVars]}, + disconnected => {fun ?MODULE:handle_disconnect/1, []} + }. + +-spec connect(pid(), name(), ingress()) -> + {ok, pid()} | {error, _Reason}. +connect(Pid, Name, Ingress) -> + case emqtt:connect(Pid) of + {ok, _Props} -> + case subscribe_remote_topic(Pid, Ingress) of + {ok, _, _RCs} -> + {ok, Pid}; + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "ingress_client_subscribe_failed", + ingress => Ingress, + reason => Reason + }), + _ = catch emqtt:stop(Pid), + Error + end; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "ingress_client_connect_failed", + reason => Reason, + name => Name + }), + _ = catch emqtt:stop(Pid), + Error + end. + +subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> + emqtt:subscribe(Pid, RemoteTopic, QoS). + +%% + +-spec config(map(), name()) -> + ingress(). +config(#{remote := RC, local := LC} = Conf, BridgeName) -> + Conf#{ + remote => parse_remote(RC, BridgeName), + local => emqx_connector_mqtt_msg:parse(LC) + }. + +parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> + QoS = downgrade_ingress_qos(QoSIn), + case QoS of + QoSIn -> + ok; + _ -> + ?SLOG(warning, #{ + msg => "downgraded_unsupported_ingress_qos", + qos_configured => QoSIn, + qos_used => QoS, + name => BridgeName + }) + end, + Conf#{qos => QoS}. + +downgrade_ingress_qos(2) -> + 1; +downgrade_ingress_qos(QoS) -> + QoS. + +%% + +-spec info(pid()) -> + [{atom(), term()}]. +info(Pid) -> + emqtt:info(Pid). + +-spec status(pid()) -> + emqx_resource:resource_status(). +status(Pid) -> + try + case proplists:get_value(socket, info(Pid)) of + Socket when Socket /= undefined -> + connected; + undefined -> + connecting + end + catch + exit:{noproc, _} -> + disconnected + end. + +%% + +handle_publish(#{properties := Props} = MsgIn, OnMessage, LocalPublish, IngressVars) -> + Msg = import_msg(MsgIn, IngressVars), + ?SLOG(debug, #{ + msg => "publish_local", + message => Msg + }), + maybe_on_message_received(Msg, OnMessage), + maybe_publish_local(Msg, LocalPublish, Props). + +handle_disconnect(_Reason) -> + ok. + +maybe_on_message_received(Msg, {Mod, Func, Args}) -> + erlang:apply(Mod, Func, [Msg | Args]); +maybe_on_message_received(_Msg, undefined) -> + ok. + +maybe_publish_local(Msg, Local = #{}, Props) -> + emqx_broker:publish(to_broker_msg(Msg, Local, Props)); +maybe_publish_local(_Msg, undefined, _Props) -> + ok. + +%% + +import_msg( + #{ + dup := Dup, + payload := Payload, + properties := Props, + qos := QoS, + retain := Retain, + topic := Topic + }, + #{server := Server} +) -> + #{ + id => emqx_guid:to_hexstr(emqx_guid:gen()), + server => Server, + payload => Payload, + topic => Topic, + qos => QoS, + dup => Dup, + retain => Retain, + pub_props => printable_maps(Props), + message_received_at => erlang:system_time(millisecond) + }. + +printable_maps(undefined) -> + #{}; +printable_maps(Headers) -> + maps:fold( + fun + ('User-Property', V0, AccIn) when is_list(V0) -> + AccIn#{ + 'User-Property' => maps:from_list(V0), + 'User-Property-Pairs' => [ + #{ + key => Key, + value => Value + } + || {Key, Value} <- V0 + ] + }; + (K, V0, AccIn) -> + AccIn#{K => V0} + end, + #{}, + Headers + ). + +%% published from remote node over a MQTT connection +to_broker_msg(Msg, Vars, undefined) -> + to_broker_msg(Msg, Vars, #{}); +to_broker_msg(#{dup := Dup} = Msg, Local, Props) -> + #{ + topic := Topic, + payload := Payload, + qos := QoS, + retain := Retain + } = emqx_connector_mqtt_msg:render(Msg, Local), + PubProps = maps:get(pub_props, Msg, #{}), + emqx_message:set_headers( + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, + emqx_message:set_flags( + #{dup => Dup, retain => Retain}, + emqx_message:make(bridge, QoS, Topic, Payload) + ) + ). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl new file mode 100644 index 000000000..b57d69df6 --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -0,0 +1,95 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_msg). + +-export([parse/1]). +-export([render/2]). + +-export_type([msgvars/0]). + +-type template() :: emqx_plugin_libs_rule:tmpl_token(). + +-type msgvars() :: #{ + topic => template(), + qos => template() | emqx_types:qos(), + retain => template() | boolean(), + payload => template() | undefined +}. + +%% + +-spec parse(#{ + topic => iodata(), + qos => iodata() | emqx_types:qos(), + retain => iodata() | boolean(), + payload => iodata() +}) -> + msgvars(). +parse(Conf) -> + Acc1 = parse_field(topic, Conf, Conf), + Acc2 = parse_field(qos, Conf, Acc1), + Acc3 = parse_field(payload, Conf, Acc2), + parse_field(retain, Conf, Acc3). + +parse_field(Key, Conf, Acc) -> + case Conf of + #{Key := Val} when is_binary(Val) -> + Acc#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)}; + #{Key := Val} -> + Acc#{Key => Val}; + #{} -> + Acc + end. + +render( + Msg, + #{ + topic := TopicToken, + qos := QoSToken, + retain := RetainToken + } = Vars +) -> + #{ + topic => render_string(TopicToken, Msg), + payload => render_payload(Vars, Msg), + qos => render_simple_var(QoSToken, Msg), + retain => render_simple_var(RetainToken, Msg) + }. + +render_payload(From, MapMsg) -> + do_render_payload(maps:get(payload, From, undefined), MapMsg). + +do_render_payload(undefined, Msg) -> + emqx_utils_json:encode(Msg); +do_render_payload(Tks, Msg) -> + render_string(Tks, Msg). + +%% Replace a string contains vars to another string in which the placeholders are replace by the +%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: +%% "a: 1". +render_string(Tokens, Data) when is_list(Tokens) -> + emqx_placeholder:proc_tmpl(Tokens, Data, #{return => full_binary}); +render_string(Val, _Data) -> + Val. + +%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result +%% value will be an integer 1. +render_simple_var(Tokens, Data) when is_list(Tokens) -> + [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), + Var; +render_simple_var(Val, _Data) -> + Val. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index f02fd19ad..6d06029b9 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -73,7 +73,6 @@ fields("server_configs") -> } )}, {server, emqx_schema:servers_sc(#{desc => ?DESC("server")}, ?MQTT_HOST_OPTS)}, - {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {clientid_prefix, mk(binary(), #{required => false, desc => ?DESC("clientid_prefix")})}, {reconnect_interval, mk(string(), #{deprecated => {since, "v5.0.16"}})}, {proto_ver, @@ -135,12 +134,13 @@ fields("server_configs") -> ] ++ emqx_connector_schema_lib:ssl_fields(); fields("ingress") -> [ - {"remote", + {pool_size, fun ingress_pool_size/1}, + {remote, mk( ref(?MODULE, "ingress_remote"), #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_remote")} )}, - {"local", + {local, mk( ref(?MODULE, "ingress_local"), #{ @@ -206,7 +206,8 @@ fields("ingress_local") -> ]; fields("egress") -> [ - {"local", + {pool_size, fun egress_pool_size/1}, + {local, mk( ref(?MODULE, "egress_local"), #{ @@ -214,7 +215,7 @@ fields("egress") -> required => false } )}, - {"remote", + {remote, mk( ref(?MODULE, "egress_remote"), #{ @@ -274,6 +275,16 @@ fields("egress_remote") -> )} ]. +ingress_pool_size(desc) -> + ?DESC("ingress_pool_size"); +ingress_pool_size(Prop) -> + emqx_connector_schema_lib:pool_size(Prop). + +egress_pool_size(desc) -> + ?DESC("egress_pool_size"); +egress_pool_size(Prop) -> + emqx_connector_schema_lib:pool_size(Prop). + desc("server_configs") -> ?DESC("server_configs"); desc("ingress") -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl deleted file mode 100644 index 223d4d058..000000000 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ /dev/null @@ -1,490 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_mqtt_worker). - --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx.hrl"). - -%% APIs --export([ - init/2, - connect/1, - stop/1 -]). - -%% management APIs --export([ - status/1, - ping/1, - info/1, - send_to_remote/3, - send_to_remote_async/4 -]). - --export([handle_publish/4]). --export([handle_disconnect/1]). - --export_type([config/0]). - --type template() :: emqx_plugin_libs_rule:tmpl_token(). - --type name() :: term(). --type options() :: #{ - % endpoint - server := iodata(), - pool_size := pos_integer(), - % emqtt client options - proto_ver := v3 | v4 | v5, - username := binary(), - password := binary(), - clientid := binary(), - clean_start := boolean(), - max_inflight := pos_integer(), - connect_timeout := pos_integer(), - retry_interval := timeout(), - keepalive := non_neg_integer(), - bridge_mode := boolean(), - ssl := boolean(), - ssl_opts := proplists:proplist(), - % bridge options - ingress := map(), - egress := map() -}. - --type client_option() :: - emqtt:option() - | {pool_size, pos_integer()} - | {name, name()} - | {ingress, ingress() | undefined}. - --type config() :: egress() | undefined. - --type ingress() :: #{ - remote := #{ - topic := emqx_topic:topic(), - qos => emqx_types:qos() - }, - local := msgvars(), - on_message_received := {module(), atom(), [term()]} -}. - --type egress() :: #{ - local => #{ - topic => emqx_topic:topic() - }, - remote := msgvars() -}. - --type msgvars() :: #{ - topic => template(), - qos => template() | emqx_types:qos(), - retain => template() | boolean(), - payload => template() | undefined -}. - --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - --spec init(name(), options()) -> - {ok, [client_option()], config()}. -init(Name, BridgeOpts) -> - Ingress = pre_process_ingress(maps:get(ingress, BridgeOpts), Name, BridgeOpts), - Egress = pre_process_egress(maps:get(egress, BridgeOpts)), - ClientOpts = mk_client_options(Name, Ingress, BridgeOpts), - {ok, maps:to_list(ClientOpts), Egress}. - -%% @doc Start a bridge worker. --spec connect([client_option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(ClientOpts0) -> - ?SLOG(debug, #{ - msg => "client_starting", - options => emqx_utils:redact(ClientOpts0) - }), - {value, {_, Name}, ClientOpts1} = lists:keytake(name, 1, ClientOpts0), - {value, {_, WorkerId}, ClientOpts} = lists:keytake(ecpool_worker_id, 1, ClientOpts1), - case emqtt:start_link(mk_emqtt_opts(WorkerId, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name, WorkerId, ClientOpts); - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "client_start_failed", - config => emqx_utils:redact(ClientOpts), - reason => Reason - }), - Error - end. - -mk_emqtt_opts(WorkerId, ClientOpts) -> - ClientId = proplists:get_value(clientid, ClientOpts), - lists:keystore(clientid, 1, ClientOpts, {clientid, mk_clientid(WorkerId, ClientId)}). - -mk_clientid(WorkerId, ClientId) -> - iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). - -connect(Pid, Name, WorkerId, ClientOpts) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - Ingress = proplists:get_value(ingress, ClientOpts), - case subscribe_remote_topic(Pid, WorkerId, Ingress) of - false -> - {ok, Pid}; - {ok, _, _RCs} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "client_subscribe_failed", - ingress => Ingress, - reason => Reason - }), - _ = catch emqtt:stop(Pid), - Error - end; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error - end. - -subscribe_remote_topic(Pid, WorkerId, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> - case emqx_topic:parse(RemoteTopic) of - {_Filter, #{share := _Name}} -> - % NOTE: this is shared subscription, each worker may subscribe - emqtt:subscribe(Pid, RemoteTopic, QoS); - {_Filter, #{}} when WorkerId =:= 1 -> - % NOTE: this is regular subscription, only the first worker should subscribe - emqtt:subscribe(Pid, RemoteTopic, QoS); - {_Filter, #{}} -> - false - end; -subscribe_remote_topic(_Ref, _, undefined) -> - false. - -mk_client_options(Name, Ingress, BridgeOpts) -> - Server = iolist_to_binary(maps:get(server, BridgeOpts)), - HostPort = emqx_connector_mqtt_schema:parse_server(Server), - CleanStart = - case Ingress of - #{remote := _} -> - maps:get(clean_start, BridgeOpts); - undefined -> - %% NOTE - %% We are ignoring the user configuration here because there's currently no reliable way - %% to ensure proper session recovery according to the MQTT spec. - true - end, - Opts = maps:with( - [ - pool_size, - proto_ver, - username, - password, - clientid, - max_inflight, - connect_timeout, - retry_interval, - keepalive, - bridge_mode, - ssl, - ssl_opts - ], - BridgeOpts - ), - Opts#{ - name => Name, - ingress => Ingress, - msg_handler => mk_client_event_handler(Ingress, #{server => Server}), - hosts => [HostPort], - clean_start => CleanStart, - force_ping => true - }. - -mk_client_event_handler(Ingress = #{}, Opts) -> - OnMessage = maps:get(on_message_received, Ingress, undefined), - LocalPublish = - case Ingress of - #{local := Local = #{topic := _}} -> - Local; - #{} -> - undefined - end, - #{ - publish => {fun ?MODULE:handle_publish/4, [OnMessage, LocalPublish, Opts]}, - disconnected => {fun ?MODULE:handle_disconnect/1, []} - }; -mk_client_event_handler(undefined, _Opts) -> - undefined. - -stop(Pid) -> - emqtt:stop(Pid). - -info(Pid) -> - emqtt:info(Pid). - -status(Pid) -> - try - case proplists:get_value(socket, info(Pid)) of - Socket when Socket /= undefined -> - connected; - undefined -> - connecting - end - catch - exit:{noproc, _} -> - disconnected - end. - -ping(Pid) -> - emqtt:ping(Pid). - -send_to_remote(Pid, MsgIn, Conf) -> - do_send(Pid, export_msg(MsgIn, Conf)). - -do_send(Pid, Msg) when Msg /= undefined -> - emqtt:publish(Pid, Msg); -do_send(_Name, undefined) -> - ok. - -send_to_remote_async(Pid, MsgIn, Callback, Conf) -> - do_send_async(Pid, export_msg(MsgIn, Conf), Callback). - -do_send_async(Pid, Msg, Callback) when Msg /= undefined -> - ok = emqtt:publish_async(Pid, Msg, _Timeout = infinity, Callback), - {ok, Pid}; -do_send_async(_Pid, undefined, _Callback) -> - ok. - -pre_process_ingress( - #{remote := RC, local := LC} = Conf, - BridgeName, - BridgeOpts -) when is_map(Conf) -> - Conf#{ - remote => pre_process_in_remote(RC, BridgeName, BridgeOpts), - local => pre_process_common(LC) - }; -pre_process_ingress(Conf, _, _) when is_map(Conf) -> - %% have no 'local' field in the config - undefined. - -pre_process_egress(#{remote := RC} = Conf) when is_map(Conf) -> - Conf#{remote => pre_process_common(RC)}; -pre_process_egress(Conf) when is_map(Conf) -> - %% have no 'remote' field in the config - undefined. - -pre_process_common(Conf0) -> - Conf1 = pre_process_conf(topic, Conf0), - Conf2 = pre_process_conf(qos, Conf1), - Conf3 = pre_process_conf(payload, Conf2), - pre_process_conf(retain, Conf3). - -pre_process_conf(Key, Conf) -> - case maps:find(Key, Conf) of - error -> - Conf; - {ok, Val} when is_binary(Val) -> - Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)}; - {ok, Val} -> - Conf#{Key => Val} - end. - -pre_process_in_remote(#{qos := QoSIn} = Conf, BridgeName, BridgeOpts) -> - QoS = downgrade_ingress_qos(QoSIn), - case QoS of - QoSIn -> - ok; - _ -> - ?SLOG(warning, #{ - msg => "downgraded_unsupported_ingress_qos", - qos_configured => QoSIn, - qos_used => QoS, - name => BridgeName, - options => BridgeOpts - }) - end, - Conf#{qos => QoS}. - -downgrade_ingress_qos(2) -> - 1; -downgrade_ingress_qos(QoS) -> - QoS. - -export_msg(Msg, #{remote := Remote}) -> - to_remote_msg(Msg, Remote); -export_msg(Msg, undefined) -> - ?SLOG(error, #{ - msg => "forwarding_unavailable", - message => Msg, - reason => "egress is not configured" - }), - undefined. - -%% - -handle_publish(#{properties := Props} = MsgIn, OnMessage, LocalPublish, Opts) -> - Msg = import_msg(MsgIn, Opts), - ?SLOG(debug, #{ - msg => "publish_local", - message => Msg - }), - maybe_on_message_received(Msg, OnMessage), - maybe_publish_local(Msg, LocalPublish, Props). - -handle_disconnect(_Reason) -> - ok. - -maybe_on_message_received(Msg, {Mod, Func, Args}) -> - erlang:apply(Mod, Func, [Msg | Args]); -maybe_on_message_received(_Msg, undefined) -> - ok. - -maybe_publish_local(Msg, Local = #{}, Props) -> - emqx_broker:publish(to_broker_msg(Msg, Local, Props)); -maybe_publish_local(_Msg, undefined, _Props) -> - ok. - -import_msg( - #{ - dup := Dup, - payload := Payload, - properties := Props, - qos := QoS, - retain := Retain, - topic := Topic - }, - #{server := Server} -) -> - #{ - id => emqx_guid:to_hexstr(emqx_guid:gen()), - server => Server, - payload => Payload, - topic => Topic, - qos => QoS, - dup => Dup, - retain => Retain, - pub_props => printable_maps(Props), - message_received_at => erlang:system_time(millisecond) - }. - -printable_maps(undefined) -> - #{}; -printable_maps(Headers) -> - maps:fold( - fun - ('User-Property', V0, AccIn) when is_list(V0) -> - AccIn#{ - 'User-Property' => maps:from_list(V0), - 'User-Property-Pairs' => [ - #{ - key => Key, - value => Value - } - || {Key, Value} <- V0 - ] - }; - (K, V0, AccIn) -> - AccIn#{K => V0} - end, - #{}, - Headers - ). - -%% Shame that we have to know the callback module here -%% would be great if we can get rid of #mqtt_msg{} record -%% and use #message{} in all places. --spec to_remote_msg(emqx_types:message() | map(), msgvars()) -> - #mqtt_msg{}. -to_remote_msg(#message{flags = Flags} = Msg, Vars) -> - {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), - to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars); -to_remote_msg( - MapMsg, - #{ - topic := TopicToken, - qos := QoSToken, - retain := RetainToken - } = Remote -) when is_map(MapMsg) -> - Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = process_payload(Remote, MapMsg), - QoS = replace_simple_var(QoSToken, MapMsg), - Retain = replace_simple_var(RetainToken, MapMsg), - PubProps = maps:get(pub_props, MapMsg, #{}), - #mqtt_msg{ - qos = QoS, - retain = Retain, - topic = Topic, - props = emqx_utils:pub_props_to_packet(PubProps), - payload = Payload - }. - -%% published from remote node over a MQTT connection -to_broker_msg(Msg, Vars, undefined) -> - to_broker_msg(Msg, Vars, #{}); -to_broker_msg( - #{dup := Dup} = MapMsg, - #{ - topic := TopicToken, - qos := QoSToken, - retain := RetainToken - } = Local, - Props -) -> - Topic = replace_vars_in_str(TopicToken, MapMsg), - Payload = process_payload(Local, MapMsg), - QoS = replace_simple_var(QoSToken, MapMsg), - Retain = replace_simple_var(RetainToken, MapMsg), - PubProps = maps:get(pub_props, MapMsg, #{}), - set_headers( - Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, - emqx_message:set_flags( - #{dup => Dup, retain => Retain}, - emqx_message:make(bridge, QoS, Topic, Payload) - ) - ). - -process_payload(From, MapMsg) -> - do_process_payload(maps:get(payload, From, undefined), MapMsg). - -do_process_payload(undefined, Msg) -> - emqx_utils_json:encode(Msg); -do_process_payload(Tks, Msg) -> - replace_vars_in_str(Tks, Msg). - -%% Replace a string contains vars to another string in which the placeholders are replace by the -%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: -%% "a: 1". -replace_vars_in_str(Tokens, Data) when is_list(Tokens) -> - emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary}); -replace_vars_in_str(Val, _Data) -> - Val. - -%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result -%% value will be an integer 1. -replace_simple_var(Tokens, Data) when is_list(Tokens) -> - [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), - Var; -replace_simple_var(Val, _Data) -> - Val. - -set_headers(Val, Msg) -> - emqx_message:set_headers(Val, Msg). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 840c6cfec..37d7b1696 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -121,7 +121,8 @@ -export_type([ resource_id/0, - resource_data/0 + resource_data/0, + resource_status/0 ]). -optional_callbacks([ diff --git a/rel/i18n/emqx_connector_mqtt_schema.hocon b/rel/i18n/emqx_connector_mqtt_schema.hocon index e37e87e49..509fc4209 100644 --- a/rel/i18n/emqx_connector_mqtt_schema.hocon +++ b/rel/i18n/emqx_connector_mqtt_schema.hocon @@ -32,6 +32,14 @@ is configured, then both the data got from the rule and the MQTT messages that m egress_desc.label: """Egress Configs""" +egress_pool_size.desc: +"""Size of the pool of MQTT clients that will publish messages to the remote broker.
+ Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:egress:${node}:${n}' + where 'n' is the number of a client inside the pool.""" + +egress_pool_size.label: +"""Pool Size""" + egress_local.desc: """The configs about receiving messages from local broker.""" @@ -75,6 +83,16 @@ ingress_desc.desc: ingress_desc.label: """Ingress Configs""" +ingress_pool_size.desc: +"""Size of the pool of MQTT clients that will ingest messages from the remote broker.
+ This value will be respected only if 'remote.topic' is a shared subscription topic filter, + otherwise only a single MQTT client will be used. + Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:ingress:${node}:${n}' + where 'n' is the number of a client inside the pool.""" + +ingress_pool_size.label: +"""Pool Size""" + ingress_local.desc: """The configs about sending message to the local broker.""" From eed9358abdb4b2ebfa9a86cfb27408488c51a147 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 21:04:21 +0300 Subject: [PATCH 10/17] chore: bump `ecpool` to 0.5.4 With fixed typings and empty pool handling. --- .../emqx_connector/src/emqx_connector_mqtt.erl | 18 ++++++------------ mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index a75f4db39..255247011 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -162,7 +162,7 @@ on_query( #{egress_pool_name := PoolName, egress_config := Config} ) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - handle_send_result(with_worker(PoolName, send, [Msg, Config])); + handle_send_result(with_egress_client(PoolName, send, [Msg, Config])); on_query(ResourceId, {send_message, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", @@ -179,7 +179,7 @@ on_query_async( ) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - Result = with_worker(PoolName, send_async, [Msg, Callback, Config]), + Result = with_egress_client(PoolName, send_async, [Msg, Callback, Config]), case Result of ok -> ok; @@ -196,16 +196,8 @@ on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> reason => "Egress is not configured" }). -with_worker(ResourceId, Fun, Args) -> - Worker = ecpool:get_client(ResourceId), - case is_pid(Worker) andalso ecpool_worker:client(Worker) of - {ok, Client} -> - erlang:apply(emqx_connector_mqtt_egress, Fun, [Client | Args]); - {error, Reason} -> - {error, Reason}; - false -> - {error, disconnected} - end. +with_egress_client(ResourceId, Fun, Args) -> + ecpool:pick_and_do(ResourceId, {emqx_connector_mqtt_egress, Fun, Args}, no_handover). on_async_result(Callback, Result) -> apply_callback_function(Callback, handle_send_result(Result)). @@ -233,6 +225,8 @@ classify_reply(Reply = #{reason_code := _}) -> classify_error(disconnected = Reason) -> {recoverable_error, Reason}; +classify_error(ecpool_empty) -> + {recoverable_error, disconnected}; classify_error({disconnected, _RC, _} = Reason) -> {recoverable_error, Reason}; classify_error({shutdown, _} = Reason) -> diff --git a/mix.exs b/mix.exs index 2e2882e15..3fa3cc2bc 100644 --- a/mix.exs +++ b/mix.exs @@ -59,7 +59,7 @@ defmodule EMQXUmbrella.MixProject do {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.10", override: true}, - {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, # maybe forbid to fetch quicer diff --git a/rebar.config b/rebar.config index 81545a7be..6a44b8074 100644 --- a/rebar.config +++ b/rebar.config @@ -66,7 +66,7 @@ , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.10"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} From b9021dfa309c59252f2bf15b42e44f8b1b401318 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 21:17:35 +0300 Subject: [PATCH 11/17] chore: drop orphaned i18n file --- rel/i18n/emqx_connector_mqtt.hocon | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 rel/i18n/emqx_connector_mqtt.hocon diff --git a/rel/i18n/emqx_connector_mqtt.hocon b/rel/i18n/emqx_connector_mqtt.hocon deleted file mode 100644 index 80303b825..000000000 --- a/rel/i18n/emqx_connector_mqtt.hocon +++ /dev/null @@ -1,21 +0,0 @@ -emqx_connector_mqtt { - -name.desc: -"""Connector name, used as a human-readable description of the connector.""" - -name.label: -"""Connector Name""" - -num_of_bridges.desc: -"""The current number of bridges that are using this connector.""" - -num_of_bridges.label: -"""Num of Bridges""" - -type.desc: -"""The Connector Type.""" - -type.label: -"""Connector Type""" - -} From ebd612b194bae1a3b94f875e743050db760df71f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 30 May 2023 14:51:53 +0300 Subject: [PATCH 12/17] chore: bump applications versions * emqx_connector 0.1.25 * emqx_rule_engine 5.0.19 * emqx_ee_bridge 0.1.15 --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index e9f79723a..283c27f31 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.24"}, + {vsn, "0.1.25"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index c6f94f5ea..7b4d1ee98 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.18"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 4a52b2c35..4b6acf915 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {applications, [ kernel, From 7e7b50c5ba3b862834f72e75e375c6b2f0c7d5af Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 22:00:38 +0300 Subject: [PATCH 13/17] refactor(mqttbridge): move into separate application --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 8 ++++---- .../src/emqx_bridge_mqtt.app.src | 18 ++++++++++++++++++ .../src/emqx_bridge_mqtt_connector.erl} | 16 ++++++++-------- .../src/emqx_bridge_mqtt_connector_schema.erl} | 10 +++++----- .../src/emqx_bridge_mqtt_egress.erl} | 10 +++++----- .../src/emqx_bridge_mqtt_ingress.erl} | 8 ++++---- .../src/emqx_bridge_mqtt_msg.erl} | 2 +- .../src}/emqx_bridge_mqtt_schema.erl | 2 +- .../test/emqx_bridge_mqtt_SUITE.erl | 11 +++++------ mix.exs | 1 + rebar.config.erl | 1 + ...=> emqx_bridge_mqtt_connector_schema.hocon} | 2 +- 12 files changed, 54 insertions(+), 35 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src rename apps/{emqx_connector/src/emqx_connector_mqtt.erl => emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl} (95%) rename apps/{emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl => emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl} (95%) rename apps/{emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl => emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl} (94%) rename apps/{emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl => emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl} (97%) rename apps/{emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl => emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl} (98%) rename apps/{emqx_bridge/src/schema => emqx_bridge_mqtt/src}/emqx_bridge_mqtt_schema.erl (97%) rename apps/{emqx_bridge => emqx_bridge_mqtt}/test/emqx_bridge_mqtt_SUITE.erl (99%) rename rel/i18n/{emqx_connector_mqtt_schema.hocon => emqx_bridge_mqtt_connector_schema.hocon} (99%) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index fd67c622c..1a3edb484 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -58,14 +58,14 @@ ). -if(?EMQX_RELEASE_EDITION == ee). -bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; -bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; +bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; +bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; bridge_to_resource_type(webhook) -> emqx_connector_http; bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType). -else. -bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; -bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; +bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; +bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; bridge_to_resource_type(webhook) -> emqx_connector_http. -endif. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src new file mode 100644 index 000000000..54d7ffbed --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -0,0 +1,18 @@ +%% -*- mode: erlang -*- +{application, emqx_bridge_mqtt, [ + {description, "EMQX MQTT Broker Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + emqx, + emqx_resource, + emqx_bridge, + emqtt + ]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl similarity index 95% rename from apps/emqx_connector/src/emqx_connector_mqtt.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 255247011..13be52a0c 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mqtt). +-module(emqx_bridge_mqtt_connector). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -83,7 +83,7 @@ start_ingress(ResourceId, Ingress, ClientOpts) -> {ingress, Ingress}, {client_opts, ClientOpts} ], - case emqx_resource_pool:start(PoolName, emqx_connector_mqtt_ingress, Options) of + case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_ingress, Options) of ok -> {ok, #{ingress_pool_name => PoolName}}; {error, {start_pool_failed, _, Reason}} -> @@ -128,11 +128,11 @@ start_egress(ResourceId, Egress, ClientOpts) -> {pool_size, PoolSize}, {client_opts, ClientOpts} ], - case emqx_resource_pool:start(PoolName, emqx_connector_mqtt_egress, Options) of + case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_egress, Options) of ok -> {ok, #{ egress_pool_name => PoolName, - egress_config => emqx_connector_mqtt_egress:config(Egress) + egress_config => emqx_bridge_mqtt_egress:config(Egress) }}; {error, {start_pool_failed, _, Reason}} -> {error, Reason} @@ -197,7 +197,7 @@ on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> }). with_egress_client(ResourceId, Fun, Args) -> - ecpool:pick_and_do(ResourceId, {emqx_connector_mqtt_egress, Fun, Args}, no_handover). + ecpool:pick_and_do(ResourceId, {emqx_bridge_mqtt_egress, Fun, Args}, no_handover). on_async_result(Callback, Result) -> apply_callback_function(Callback, handle_send_result(Result)). @@ -250,9 +250,9 @@ on_get_status(_ResourceId, State) -> get_status({Pool, Worker}) -> case ecpool_worker:client(Worker) of {ok, Client} when Pool == ingress_pool_name -> - emqx_connector_mqtt_ingress:status(Client); + emqx_bridge_mqtt_ingress:status(Client); {ok, Client} when Pool == egress_pool_name -> - emqx_connector_mqtt_egress:status(Client); + emqx_bridge_mqtt_egress:status(Client); {error, _} -> disconnected end. @@ -300,7 +300,7 @@ mk_client_opts( ssl := #{enable := EnableSsl} = Ssl } ) -> - HostPort = emqx_connector_mqtt_schema:parse_server(Server), + HostPort = emqx_bridge_mqtt_connector_schema:parse_server(Server), Options = maps:with( [ proto_ver, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl similarity index 95% rename from apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 6d06029b9..1dc3ca5f8 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mqtt_schema). +-module(emqx_bridge_mqtt_connector_schema). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -138,13 +138,13 @@ fields("ingress") -> {remote, mk( ref(?MODULE, "ingress_remote"), - #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_remote")} + #{desc => ?DESC("ingress_remote")} )}, {local, mk( ref(?MODULE, "ingress_local"), #{ - desc => ?DESC(emqx_connector_mqtt_schema, "ingress_local") + desc => ?DESC("ingress_local") } )} ]; @@ -211,7 +211,7 @@ fields("egress") -> mk( ref(?MODULE, "egress_local"), #{ - desc => ?DESC(emqx_connector_mqtt_schema, "egress_local"), + desc => ?DESC("egress_local"), required => false } )}, @@ -219,7 +219,7 @@ fields("egress") -> mk( ref(?MODULE, "egress_remote"), #{ - desc => ?DESC(emqx_connector_mqtt_schema, "egress_remote"), + desc => ?DESC("egress_remote"), required => true } )} diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl similarity index 94% rename from apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 0e413cbc9..673a30726 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mqtt_egress). +-module(emqx_bridge_mqtt_egress). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -51,7 +51,7 @@ local => #{ topic => emqx_topic:topic() }, - remote := emqx_connector_mqtt_msg:msgvars() + remote := emqx_bridge_mqtt_msg:msgvars() }. %% @doc Start an ingress bridge worker. @@ -102,7 +102,7 @@ connect(Pid, Name) -> -spec config(map()) -> egress(). config(#{remote := RC = #{}} = Conf) -> - Conf#{remote => emqx_connector_mqtt_msg:parse(RC)}. + Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}. -spec send(pid(), message(), egress()) -> ok. @@ -118,7 +118,7 @@ send_async(Pid, MsgIn, Callback, Egress) -> export_msg(Msg, #{remote := Remote}) -> to_remote_msg(Msg, Remote). --spec to_remote_msg(message(), emqx_connector_mqtt_msg:msgvars()) -> +-spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars()) -> remote_message(). to_remote_msg(#message{flags = Flags} = Msg, Vars) -> {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), @@ -129,7 +129,7 @@ to_remote_msg(Msg = #{}, Remote) -> payload := Payload, qos := QoS, retain := Retain - } = emqx_connector_mqtt_msg:render(Msg, Remote), + } = emqx_bridge_mqtt_msg:render(Msg, Remote), PubProps = maps:get(pub_props, Msg, #{}), #mqtt_msg{ qos = QoS, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl similarity index 97% rename from apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index c11895c49..78bbf7753 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mqtt_ingress). +-module(emqx_bridge_mqtt_ingress). -include_lib("emqx/include/logger.hrl"). @@ -46,7 +46,7 @@ topic := emqx_topic:topic(), qos => emqx_types:qos() }, - local := emqx_connector_mqtt_msg:msgvars(), + local := emqx_bridge_mqtt_msg:msgvars(), on_message_received := {module(), atom(), [term()]} }. @@ -135,7 +135,7 @@ subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> config(#{remote := RC, local := LC} = Conf, BridgeName) -> Conf#{ remote => parse_remote(RC, BridgeName), - local => emqx_connector_mqtt_msg:parse(LC) + local => emqx_bridge_mqtt_msg:parse(LC) }. parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> @@ -261,7 +261,7 @@ to_broker_msg(#{dup := Dup} = Msg, Local, Props) -> payload := Payload, qos := QoS, retain := Retain - } = emqx_connector_mqtt_msg:render(Msg, Local), + } = emqx_bridge_mqtt_msg:render(Msg, Local), PubProps = maps:get(pub_props, Msg, #{}), emqx_message:set_headers( Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl similarity index 98% rename from apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl index b57d69df6..8a8cffe55 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mqtt_msg). +-module(emqx_bridge_mqtt_msg). -export([parse/1]). -export([render/2]). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl similarity index 97% rename from apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl rename to apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl index 5cd1693c7..a312dfaa9 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -42,7 +42,7 @@ fields("config") -> } )} ] ++ - emqx_connector_mqtt_schema:fields("config"); + emqx_bridge_mqtt_connector_schema:fields("config"); fields("creation_opts") -> Opts = emqx_resource_schema:fields("creation_opts"), [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)]; diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl similarity index 99% rename from apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl rename to apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6c36e08e7..81f9a3573 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -130,13 +130,11 @@ suite() -> init_per_suite(Config) -> _ = application:load(emqx_conf), - %% some testcases (may from other app) already get emqx_connector started - _ = application:stop(emqx_resource), - _ = application:stop(emqx_connector), ok = emqx_common_test_helpers:start_apps( [ emqx_rule_engine, emqx_bridge, + emqx_bridge_mqtt, emqx_dashboard ], fun set_special_configs/1 @@ -150,9 +148,10 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([ - emqx_rule_engine, + emqx_dashboard, + emqx_bridge_mqtt, emqx_bridge, - emqx_dashboard + emqx_rule_engine ]), ok. @@ -307,7 +306,7 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> emqx_resource_manager:lookup_cached(ResourceID), ClientInfo = ecpool:pick_and_do( EgressPoolName, - {emqx_connector_mqtt_egress, info, []}, + {emqx_bridge_mqtt_egress, info, []}, no_handover ), ?assertMatch( diff --git a/mix.exs b/mix.exs index 3fa3cc2bc..7983a7482 100644 --- a/mix.exs +++ b/mix.exs @@ -372,6 +372,7 @@ defmodule EMQXUmbrella.MixProject do emqx_gateway_exproto: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, + emqx_bridge_mqtt: :permanent, emqx_rule_engine: :permanent, emqx_modules: :permanent, emqx_management: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index a8474a703..6c54dffca 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -426,6 +426,7 @@ relx_apps(ReleaseType, Edition) -> emqx_gateway_exproto, emqx_exhook, emqx_bridge, + emqx_bridge_mqtt, emqx_rule_engine, emqx_modules, emqx_management, diff --git a/rel/i18n/emqx_connector_mqtt_schema.hocon b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon similarity index 99% rename from rel/i18n/emqx_connector_mqtt_schema.hocon rename to rel/i18n/emqx_bridge_mqtt_connector_schema.hocon index 509fc4209..ed7a59dcd 100644 --- a/rel/i18n/emqx_connector_mqtt_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon @@ -1,4 +1,4 @@ -emqx_connector_mqtt_schema { +emqx_bridge_mqtt_connector_schema { bridge_mode.desc: """If enable bridge mode. From 1c2719236c58d7502d6847b022c9f986051c7cca Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 May 2023 22:28:31 +0300 Subject: [PATCH 14/17] chore(mqttbridge): add README --- apps/emqx_bridge_mqtt/README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 apps/emqx_bridge_mqtt/README.md diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md new file mode 100644 index 000000000..f286913e7 --- /dev/null +++ b/apps/emqx_bridge_mqtt/README.md @@ -0,0 +1,32 @@ +# EMQX MQTT Broker Bridge + +This application connects EMQX to virtually any MQTT broker adhering to either [MQTTv3][1] or [MQTTv5][2] standard. The connection is facilitated through the _MQTT bridge_ abstraction, allowing for the flow of data in both directions: from the remote broker to EMQX (ingress) and from EMQX to the remote broker (egress). + +User can create a rule and easily ingest into a remote MQTT broker by leveraging [EMQX Rules][3]. + + +# Documentation + +- Refer to [Bridge Data into MQTT Broker][4] for how to use EMQX dashboard to set up ingress or egress bridge, or even both at the same time. + +- Refer to [EMQX Rules][3] for the EMQX rules engine introduction. + + +# HTTP APIs + +Several APIs are provided for bridge management, refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +# Contributing + +Please see our [contributing guide](../../CONTRIBUTING.md). + + +# License + +Apache License 2.0, see [LICENSE](../../APL.txt). + +[1]: https://docs.oasis-open.org/mqtt/mqtt/v3.1.1 +[2]: https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html +[3]: https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html +[4]: https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-mqtt.html From 95cc9b9b7239cdf8e089bb41f20ae771119d32ac Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 30 May 2023 17:18:31 +0300 Subject: [PATCH 15/17] fix(mqttbridge): ensure elixir release assembles successfully --- apps/emqx_bridge_mqtt/rebar.config | 3 +++ mix.exs | 1 + 2 files changed, 4 insertions(+) create mode 100644 apps/emqx_bridge_mqtt/rebar.config diff --git a/apps/emqx_bridge_mqtt/rebar.config b/apps/emqx_bridge_mqtt/rebar.config new file mode 100644 index 000000000..35ccc1a37 --- /dev/null +++ b/apps/emqx_bridge_mqtt/rebar.config @@ -0,0 +1,3 @@ +{deps, [ + {emqx, {path, "../../apps/emqx"}} +]}. diff --git a/mix.exs b/mix.exs index 7983a7482..dab11693c 100644 --- a/mix.exs +++ b/mix.exs @@ -310,6 +310,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_connector, :emqx_exhook, :emqx_bridge, + :emqx_bridge_mqtt, :emqx_modules, :emqx_management, :emqx_retainer, From 26819a647cd4524defd41d228b9e16aa9ad84eee Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 30 May 2023 18:13:24 +0300 Subject: [PATCH 16/17] fix(mqttbridge): clarify schema descriptions + log messages Co-authored-by: Zaiming (Stone) Shi --- .../src/emqx_bridge_mqtt_connector.erl | 10 +++++++--- .../src/emqx_bridge_mqtt_ingress.erl | 20 ++++++++++--------- .../emqx_bridge_mqtt_connector_schema.hocon | 13 ++++++------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 13be52a0c..60a512011 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -76,7 +76,7 @@ start_ingress(ResourceId, Conf) -> start_ingress(ResourceId, Ingress, ClientOpts) -> PoolName = <>, - PoolSize = choose_ingress_pool_size(Ingress), + PoolSize = choose_ingress_pool_size(ResourceId, Ingress), Options = [ {name, PoolName}, {pool_size, PoolSize}, @@ -90,7 +90,10 @@ start_ingress(ResourceId, Ingress, ClientOpts) -> {error, Reason} end. -choose_ingress_pool_size(#{remote := #{topic := RemoteTopic}, pool_size := PoolSize}) -> +choose_ingress_pool_size( + ResourceId, + #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} +) -> case emqx_topic:parse(RemoteTopic) of {_Filter, #{share := _Name}} -> % NOTE: this is shared subscription, many workers may subscribe @@ -98,7 +101,8 @@ choose_ingress_pool_size(#{remote := #{topic := RemoteTopic}, pool_size := PoolS {_Filter, #{}} -> % NOTE: this is regular subscription, only one worker should subscribe ?SLOG(warning, #{ - msg => "ingress_pool_size_ignored", + msg => "mqtt_bridge_ingress_pool_size_ignored", + connector => ResourceId, reason => "Remote topic filter is not a shared subscription, " "ingress pool will start with a single worker", diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index 78bbf7753..91ec27e74 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -29,7 +29,7 @@ info/1 ]). --export([handle_publish/4]). +-export([handle_publish/5]). -export([handle_disconnect/1]). -type name() :: term(). @@ -62,7 +62,7 @@ connect(Options) -> WorkerId = proplists:get_value(ecpool_worker_id, Options), Ingress = config(proplists:get_value(ingress, Options), Name), ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(WorkerId, Ingress, ClientOpts)) of + case emqtt:start_link(mk_client_opts(Name, WorkerId, Ingress, ClientOpts)) of {ok, Pid} -> connect(Pid, Name, Ingress); {error, Reason} = Error -> @@ -74,16 +74,16 @@ connect(Options) -> Error end. -mk_client_opts(WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> +mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> ClientOpts#{ clientid := mk_clientid(WorkerId, ClientId), - msg_handler => mk_client_event_handler(Ingress) + msg_handler => mk_client_event_handler(Name, Ingress) }. mk_clientid(WorkerId, ClientId) -> iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). -mk_client_event_handler(Ingress = #{}) -> +mk_client_event_handler(Name, Ingress = #{}) -> IngressVars = maps:with([server], Ingress), OnMessage = maps:get(on_message_received, Ingress, undefined), LocalPublish = @@ -94,7 +94,7 @@ mk_client_event_handler(Ingress = #{}) -> undefined end, #{ - publish => {fun ?MODULE:handle_publish/4, [OnMessage, LocalPublish, IngressVars]}, + publish => {fun ?MODULE:handle_publish/5, [Name, OnMessage, LocalPublish, IngressVars]}, disconnected => {fun ?MODULE:handle_disconnect/1, []} }. @@ -110,6 +110,7 @@ connect(Pid, Name, Ingress) -> ?SLOG(error, #{ msg => "ingress_client_subscribe_failed", ingress => Ingress, + name => Name, reason => Reason }), _ = catch emqtt:stop(Pid), @@ -182,11 +183,12 @@ status(Pid) -> %% -handle_publish(#{properties := Props} = MsgIn, OnMessage, LocalPublish, IngressVars) -> +handle_publish(#{properties := Props} = MsgIn, Name, OnMessage, LocalPublish, IngressVars) -> Msg = import_msg(MsgIn, IngressVars), ?SLOG(debug, #{ - msg => "publish_local", - message => Msg + msg => "ingress_publish_local", + message => Msg, + name => Name }), maybe_on_message_received(Msg, OnMessage), maybe_publish_local(Msg, LocalPublish, Props). diff --git a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon index ed7a59dcd..7c7bf68c9 100644 --- a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon @@ -34,8 +34,8 @@ egress_desc.label: egress_pool_size.desc: """Size of the pool of MQTT clients that will publish messages to the remote broker.
- Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:egress:${node}:${n}' - where 'n' is the number of a client inside the pool.""" +Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:egress:${node}:${n}' +where 'n' is the number of a client inside the pool.""" egress_pool_size.label: """Pool Size""" @@ -85,10 +85,11 @@ ingress_desc.label: ingress_pool_size.desc: """Size of the pool of MQTT clients that will ingest messages from the remote broker.
- This value will be respected only if 'remote.topic' is a shared subscription topic filter, - otherwise only a single MQTT client will be used. - Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:ingress:${node}:${n}' - where 'n' is the number of a client inside the pool.""" +This value will be respected only if 'remote.topic' is a shared subscription topic or topic-filter +(for example `$share/name1/topic1` or `$share/name2/topic2/#`), otherwise only a single MQTT client will be used. +Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:ingress:${node}:${n}' +where 'n' is the number of a client inside the pool. +NOTE: Non-shared subscription will not work well when EMQX is clustered.""" ingress_pool_size.label: """Pool Size""" From 42a4c0200d0c55ea1244d72c98ff7fa3cd8a8943 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 30 May 2023 22:21:52 +0300 Subject: [PATCH 17/17] chore: add changelog entry --- changes/ce/perf-10754.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ce/perf-10754.en.md diff --git a/changes/ce/perf-10754.en.md b/changes/ce/perf-10754.en.md new file mode 100644 index 000000000..ef32960be --- /dev/null +++ b/changes/ce/perf-10754.en.md @@ -0,0 +1,3 @@ +The MQTT bridge has been enhanced to utilize connection pooling and leverage available parallelism, substantially improving throughput. + +As a consequence, single MQTT bridge now uses a pool of `clientid`s to connect to the remote broker.