From f199a0f24a6a75584e337016e427c8dfd4d5a483 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 1 Dec 2023 10:20:58 +0100 Subject: [PATCH 01/28] feat: refactor MQTT bridge to source, action, and connector This commit: * refactors the MQTT V1 bridge into connector, source and action * Extends the compatibility layer so it works for sources * Fixes the MQTT bridge test suite so that all test cases passes We still need to add a HTTP API handling sources. Also, we still need to add HTTP API example schemes and examples for the MQTT connector/action/source. We should also make sure that we handle the corner cases of the MQTT V1 bridge automatic upgrade downgrade in a sufficiently good way: * An error is currently thrown when converting an MQTT V1 bridge without egress or ingress config. * If there is a source and action with the same name we will currently throw an error in the compatibility layer. * We will also throw an error when converting an MQTT V1 bridge with both ingress and egress. The above is probably the right thing to do but we have to make sure that we return a reasonable error to the user when this happens. (partly) Fixes: https://emqx.atlassian.net/browse/EMQX-11489 --- apps/emqx_bridge/src/emqx_action_info.erl | 76 ++- apps/emqx_bridge/src/emqx_bridge.erl | 8 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 14 +- apps/emqx_bridge/src/emqx_bridge_lib.erl | 3 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 543 +++++++++++++----- .../src/schema/emqx_bridge_v2_schema.erl | 42 +- .../src/emqx_bridge_mqtt_connector.erl | 404 ++++++++----- .../src/emqx_bridge_mqtt_connector_schema.erl | 69 ++- .../src/emqx_bridge_mqtt_egress.erl | 84 --- .../src/emqx_bridge_mqtt_ingress.erl | 293 ++++++---- .../emqx_bridge_mqtt_pubsub_action_info.erl | 221 +++++++ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 129 +++++ .../test/emqx_bridge_mqtt_SUITE.erl | 42 +- .../src/emqx_connector_resource.erl | 23 +- .../src/schema/emqx_connector_schema.erl | 161 ++++-- 15 files changed, 1536 insertions(+), 576 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 5ce60fe6c..d80050191 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -26,7 +26,10 @@ bridge_v1_type_to_action_type/1, bridge_v1_type_name/1, is_action_type/1, - registered_schema_modules/0, + is_source/1, + is_action/1, + registered_schema_modules_actions/0, + registered_schema_modules_sources/0, connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, bridge_v1_config_to_connector_config/2, @@ -51,19 +54,26 @@ ConnectorConfig :: map(), ActionConfig :: map() ) -> map(). %% Define this if the automatic config upgrade is not enough for the connector. --callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> map(). +-callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> + map() | {ConnectorTypeName :: atom(), map()}. %% Define this if the automatic config upgrade is not enough for the bridge. %% If you want to make use of the automatic config upgrade, you can call %% emqx_action_info:transform_bridge_v1_config_to_action_config/4 in your %% implementation and do some adjustments on the result. -callback bridge_v1_config_to_action_config(BridgeV1Config :: map(), ConnectorName :: binary()) -> - map(). + map() | {source | action, ActionTypeName :: atom(), map()} | 'none'. +-callback is_source() -> + boolean(). +-callback is_action() -> + boolean(). -optional_callbacks([ bridge_v1_type_name/0, connector_action_config_to_bridge_v1_config/2, bridge_v1_config_to_connector_config/1, - bridge_v1_config_to_action_config/2 + bridge_v1_config_to_action_config/2, + is_source/0, + is_action/0 ]). %% ==================================================================== @@ -96,7 +106,10 @@ hard_coded_action_info_modules_ee() -> -endif. hard_coded_action_info_modules_common() -> - [emqx_bridge_http_action_info]. + [ + emqx_bridge_http_action_info, + emqx_bridge_mqtt_pubsub_action_info + ]. hard_coded_action_info_modules() -> hard_coded_action_info_modules_common() ++ hard_coded_action_info_modules_ee(). @@ -178,10 +191,33 @@ is_action_type(Type) -> _ -> true end. -registered_schema_modules() -> +%% Returns true if the action is an ingress action, false otherwise. +is_source(Bin) when is_binary(Bin) -> + is_source(binary_to_existing_atom(Bin)); +is_source(Type) -> + ActionInfoMap = info_map(), + IsSourceMap = maps:get(is_source, ActionInfoMap), + maps:get(Type, IsSourceMap, false). + +%% Returns true if the action is an egress action, false otherwise. +is_action(Bin) when is_binary(Bin) -> + is_action(binary_to_existing_atom(Bin)); +is_action(Type) -> + ActionInfoMap = info_map(), + IsActionMap = maps:get(is_action, ActionInfoMap), + maps:get(Type, IsActionMap, true). + +registered_schema_modules_actions() -> InfoMap = info_map(), Schemas = maps:get(action_type_to_schema_module, InfoMap), - maps:to_list(Schemas). + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_action(Type)]. + +registered_schema_modules_sources() -> + InfoMap = info_map(), + Schemas = maps:get(action_type_to_schema_module, InfoMap), + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_source(Type)]. connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), @@ -293,7 +329,9 @@ initial_info_map() -> action_type_to_bridge_v1_type => #{}, action_type_to_connector_type => #{}, action_type_to_schema_module => #{}, - action_type_to_info_module => #{} + action_type_to_info_module => #{}, + is_source => #{}, + is_action => #{} }. get_info_map(Module) -> @@ -312,6 +350,20 @@ get_info_map(Module) -> false -> {ActionType, [ActionType]} end, + IsIngress = + case erlang:function_exported(Module, is_source, 0) of + true -> + Module:is_source(); + false -> + false + end, + IsEgress = + case erlang:function_exported(Module, is_action, 0) of + true -> + Module:is_action(); + false -> + true + end, #{ action_type_names => lists:foldl( @@ -351,5 +403,11 @@ get_info_map(Module) -> end, #{ActionType => Module}, BridgeV1Types - ) + ), + is_source => #{ + ActionType => IsIngress + }, + is_action => #{ + ActionType => IsEgress + } }. diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d26a44a1d..c7d9a2d27 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -353,7 +353,13 @@ get_metrics(Type, Name) -> case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of true -> BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), - emqx_bridge_v2:get_metrics(BridgeV2Type, Name); + try + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one(Type, Name), + emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) + catch + error:Reason -> + {error, Reason} + end; false -> {error, not_bridge_v1_compatible} end; diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index e1cd03ac2..3168ae590 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -548,9 +548,17 @@ schema("/bridges_probe") -> Id, case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(BridgeV2Type, BridgeName), - ?NO_CONTENT; + try + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( + BridgeType, BridgeName + ), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), + ok = emqx_bridge_v2:reset_metrics(ConfRootKey, BridgeV2Type, BridgeName), + ?NO_CONTENT + catch + error:Reason -> + ?BAD_REQUEST(Reason) + end; false -> ok = emqx_bridge_resource:reset_metrics( emqx_bridge_resource:resource_id(BridgeType, BridgeName) diff --git a/apps/emqx_bridge/src/emqx_bridge_lib.erl b/apps/emqx_bridge/src/emqx_bridge_lib.erl index 9386a38d3..7f74dfb2d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_lib.erl +++ b/apps/emqx_bridge/src/emqx_bridge_lib.erl @@ -82,7 +82,8 @@ external_ids(Type, Name) -> get_conf(BridgeType, BridgeName) -> case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - emqx_conf:get_raw([actions, BridgeType, BridgeName]); + ConfRootName = emqx_bridge_v2:get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_conf:get_raw([ConfRootName, BridgeType, BridgeName]); false -> undefined end. diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 723808919..66d4dc674 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -26,7 +26,8 @@ %% Note: this is strange right now, because it lives in `emqx_bridge_v2', but it shall be %% refactored into a new module/application with appropriate name. --define(ROOT_KEY, actions). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). %% Loading and unloading config when EMQX starts and stops -export([ @@ -38,7 +39,9 @@ -export([ list/0, + list/1, lookup/2, + lookup/3, create/3, %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies @@ -53,13 +56,16 @@ -export([ disable_enable/3, + disable_enable/4, health_check/2, send_message/4, query/4, start/2, reset_metrics/2, + reset_metrics/3, create_dry_run/2, - get_metrics/2 + get_metrics/2, + get_metrics/3 ]). %% On message publish hook (for local_topics) @@ -122,7 +128,8 @@ bridge_v1_stop/2, bridge_v1_start/2, %% For test cases only - bridge_v1_remove/2 + bridge_v1_remove/2, + get_conf_root_key_if_only_one/2 ]). %%==================================================================== @@ -151,19 +158,22 @@ %%==================================================================== load() -> - load_bridges(), + load_bridges(?ROOT_KEY_ACTIONS), + load_bridges(?ROOT_KEY_SOURCES), load_message_publish_hook(), ok = emqx_config_handler:add_handler(config_key_path_leaf(), emqx_bridge_v2), ok = emqx_config_handler:add_handler(config_key_path(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_leaf_sources(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_sources(), emqx_bridge_v2), ok. -load_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +load_bridges(RootName) -> + Bridges = emqx:get_config([RootName], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - install_bridge_v2(Type, Name, BridgeConf) + install_bridge_v2(RootName, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -172,19 +182,20 @@ load_bridges() -> ). unload() -> - unload_bridges(), + unload_bridges(?ROOT_KEY_ACTIONS), + unload_bridges(?ROOT_KEY_SOURCES), unload_message_publish_hook(), emqx_conf:remove_handler(config_key_path()), emqx_conf:remove_handler(config_key_path_leaf()), ok. -unload_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +unload_bridges(ConfRooKey) -> + Bridges = emqx:get_config([ConfRooKey], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - uninstall_bridge_v2(Type, Name, BridgeConf) + uninstall_bridge_v2(ConfRooKey, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -198,7 +209,12 @@ unload_bridges() -> -spec lookup(bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(Type, Name) -> - case emqx:get_raw_config([?ROOT_KEY, Type, Name], not_found) of + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +-spec lookup(sources | actions, bridge_v2_type(), bridge_v2_name()) -> + {ok, bridge_v2_info()} | {error, not_found}. +lookup(ConfRootName, Type, Name) -> + case emqx:get_raw_config([ConfRootName, Type, Name], not_found) of not_found -> {error, not_found}; #{<<"connector">> := BridgeConnector} = RawConf -> @@ -218,7 +234,7 @@ lookup(Type, Name) -> %% Find the Bridge V2 status from the ConnectorData ConnectorStatus = maps:get(status, ConnectorData, undefined), Channels = maps:get(added_channels, ConnectorData, #{}), - BridgeV2Id = id(Type, Name, BridgeConnector), + BridgeV2Id = id_with_root_name(ConfRootName, Type, Name, BridgeConnector), ChannelStatus = maps:get(BridgeV2Id, Channels, undefined), {DisplayBridgeV2Status, ErrorMsg} = case {ChannelStatus, ConnectorStatus} of @@ -245,20 +261,30 @@ lookup(Type, Name) -> -spec list() -> [bridge_v2_info()] | {error, term()}. list() -> - list_with_lookup_fun(fun lookup/2). + list_with_lookup_fun(?ROOT_KEY_ACTIONS, fun lookup/2). + +list(ConfRootKey) -> + LookupFun = fun(Type, Name) -> + lookup(ConfRootKey, Type, Name) + end, + list_with_lookup_fun(ConfRootKey, LookupFun). -spec create(bridge_v2_type(), bridge_v2_name(), map()) -> {ok, emqx_config:update_result()} | {error, any()}. create(BridgeType, BridgeName, RawConf) -> + create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, RawConf). + +create(ConfRootKey, BridgeType, BridgeName, RawConf) -> ?SLOG(debug, #{ bridge_action => create, bridge_version => 2, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_utils:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf), + root_key_path => ConfRootKey }), emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], RawConf, #{override_to => cluster} ). @@ -267,6 +293,9 @@ create(BridgeType, BridgeName, RawConf) -> remove(BridgeType, BridgeName) -> %% NOTE: This function can cause broken references from rules but it is only %% called directly from test cases. + remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +remove(ConfRootKey, BridgeType, BridgeName) -> ?SLOG(debug, #{ bridge_action => remove, bridge_version => 2, @@ -275,7 +304,7 @@ remove(BridgeType, BridgeName) -> }), case emqx_conf:remove( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], #{override_to => cluster} ) of @@ -307,7 +336,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> %% Helpers for CRUD API %%-------------------------------------------------------------------- -list_with_lookup_fun(LookupFun) -> +list_with_lookup_fun(ConfRootName, LookupFun) -> maps:fold( fun(Type, NameAndConf, Bridges) -> maps:fold( @@ -330,21 +359,24 @@ list_with_lookup_fun(LookupFun) -> ) end, [], - emqx:get_raw_config([?ROOT_KEY], #{}) + emqx:get_raw_config([ConfRootName], #{}) ). install_bridge_v2( + _RootName, _BridgeType, _BridgeName, #{enable := false} ) -> ok; install_bridge_v2( + RootName, BridgeV2Type, BridgeName, Config ) -> install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, combine_connector_and_bridge_v2_config( @@ -355,6 +387,7 @@ install_bridge_v2( ). install_bridge_v2_helper( + _RootName, _BridgeV2Type, _BridgeName, {error, Reason} = Error @@ -362,11 +395,12 @@ install_bridge_v2_helper( ?SLOG(warning, Reason), Error; install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(RootName, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), %% Create metrics for Bridge V2 ok = emqx_resource:create_metrics(BridgeV2Id), @@ -388,18 +422,45 @@ install_bridge_v2_helper( ConnectorId = emqx_connector_resource:resource_id( connector_type(BridgeV2Type), ConnectorName ), - ConfigWithTypeAndName = Config#{ - bridge_type => bin(BridgeV2Type), - bridge_name => bin(BridgeName) - }, emqx_resource_manager:add_channel( ConnectorId, BridgeV2Id, - ConfigWithTypeAndName + augment_channel_config( + RootName, + BridgeV2Type, + BridgeName, + Config + ) ), ok. +augment_channel_config( + ConfigRoot, + BridgeV2Type, + BridgeName, + Config +) -> + AugmentedConf = Config#{ + config_root => ConfigRoot, + bridge_type => bin(BridgeV2Type), + bridge_name => bin(BridgeName) + }, + case emqx_action_info:is_source(BridgeV2Type) of + true -> + BId = emqx_bridge_resource:bridge_id(BridgeV2Type, BridgeName), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + SourceHookpoint = source_hookpoint(BId), + HookPoints = [BridgeHookpoint, SourceHookpoint], + AugmentedConf#{hookpoints => HookPoints}; + false -> + AugmentedConf + end. + +source_hookpoint(BridgeId) -> + <<"$sources/", (bin(BridgeId))/binary>>. + uninstall_bridge_v2( + _ConfRootKey, _BridgeType, _BridgeName, #{enable := false} @@ -407,11 +468,12 @@ uninstall_bridge_v2( %% Already not installed ok; uninstall_bridge_v2( + ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), ok = emqx_resource_buffer_worker_sup:stop_workers(BridgeV2Id, CreationOpts), ok = emqx_resource:clear_metrics(BridgeV2Id), @@ -460,8 +522,11 @@ combine_connector_and_bridge_v2_config( -spec disable_enable(disable | enable, bridge_v2_type(), bridge_v2_name()) -> {ok, any()} | {error, any()}. disable_enable(Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> + disable_enable(?ROOT_KEY_ACTIONS, Action, BridgeType, BridgeName). + +disable_enable(ConfRootKey, Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], Action, #{override_to => cluster} ). @@ -477,10 +542,12 @@ start(BridgeV2Type, Name) -> ConnectorOpFun = fun(ConnectorType, ConnectorName) -> emqx_connector_resource:start(ConnectorType, ConnectorName) end, - connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, true). + ConfRootKey = ?ROOT_KEY_ACTIONS, + connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, true). -connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> +connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, lookup_conf(BridgeV2Type, Name), @@ -489,14 +556,16 @@ connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> ). connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, - {error, bridge_not_found} = Error, + {error, _} = Error, _ConnectorOpFun, _DoHealthCheck ) -> Error; connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, #{enable := false}, @@ -505,6 +574,7 @@ connector_operation_helper_with_conf( ) -> ok; connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, #{connector := ConnectorName}, @@ -519,7 +589,7 @@ connector_operation_helper_with_conf( {true, {error, Reason}} -> {error, Reason}; {true, ok} -> - case health_check(BridgeV2Type, Name) of + case health_check(ConfRootKey, BridgeV2Type, Name) of #{status := connected} -> ok; {error, Reason} -> @@ -536,14 +606,17 @@ connector_operation_helper_with_conf( -spec reset_metrics(bridge_v2_type(), bridge_v2_name()) -> ok | {error, not_found}. reset_metrics(Type, Name) -> - reset_metrics_helper(Type, Name, lookup_conf(Type, Name)). + reset_metrics(?ROOT_KEY_ACTIONS, Type, Name). -reset_metrics_helper(_Type, _Name, #{enable := false}) -> +reset_metrics(ConfRootKey, Type, Name) -> + reset_metrics_helper(ConfRootKey, Type, Name, lookup_conf(ConfRootKey, Type, Name)). + +reset_metrics_helper(_ConfRootKey, _Type, _Name, #{enable := false}) -> ok; -reset_metrics_helper(BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), +reset_metrics_helper(ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, BridgeV2Id); -reset_metrics_helper(_, _, _) -> +reset_metrics_helper(_, _, _, _) -> {error, not_found}. get_query_mode(BridgeV2Type, Config) -> @@ -599,7 +672,10 @@ send_message(BridgeType, BridgeName, Message, QueryOpts0) -> -spec health_check(BridgeType :: term(), BridgeName :: term()) -> #{status := emqx_resource:resource_status(), error := term()} | {error, Reason :: term()}. health_check(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of + health_check(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +health_check(ConfRootKey, BridgeType, BridgeName) -> + case lookup_conf(ConfRootKey, BridgeType, BridgeName) of #{ enable := true, connector := ConnectorName @@ -608,7 +684,7 @@ health_check(BridgeType, BridgeName) -> connector_type(BridgeType), ConnectorName ), emqx_resource_manager:channel_health_check( - ConnectorId, id(BridgeType, BridgeName, ConnectorName) + ConnectorId, id_with_root_name(ConfRootKey, BridgeType, BridgeName, ConnectorName) ); #{enable := false} -> {error, bridge_stopped}; @@ -652,13 +728,13 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> {_, ConnectorName} = emqx_connector_resource:parse_connector_id(ConnectorId), ChannelTestId = id(BridgeType, BridgeName, ConnectorName), Conf = emqx_utils_maps:unsafe_atom_key_map(BridgeV2RawConf), - ConfWithTypeAndName = Conf#{ - bridge_type => bin(BridgeType), - bridge_name => bin(BridgeName) - }, - case - emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, ConfWithTypeAndName) - of + AugmentedConf = augment_channel_config( + ?ROOT_KEY_ACTIONS, + BridgeType, + BridgeName, + Conf + ), + case emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, AugmentedConf) of {error, Reason} -> {error, Reason}; ok -> @@ -677,7 +753,10 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> -spec get_metrics(bridge_v2_type(), bridge_v2_name()) -> emqx_metrics_worker:metrics(). get_metrics(Type, Name) -> - emqx_resource:get_metrics(id(Type, Name)). + get_metrics(?ROOT_KEY_ACTIONS, Type, Name). + +get_metrics(ConfRootKey, Type, Name) -> + emqx_resource:get_metrics(id_with_root_name(ConfRootKey, Type, Name)). %%==================================================================== %% On message publish hook (for local topics) @@ -690,7 +769,7 @@ reload_message_publish_hook(Bridges) -> ok = load_message_publish_hook(Bridges). load_message_publish_hook() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), load_message_publish_hook(Bridges). load_message_publish_hook(Bridges) -> @@ -754,7 +833,7 @@ send_to_matched_egress_bridges(Topic, Msg) -> ). get_matched_egress_bridges(Topic) -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), maps:fold( fun(BType, Conf, Acc0) -> maps:fold( @@ -800,16 +879,21 @@ parse_id(Id) -> end. get_channels_for_connector(ConnectorId) -> + Actions = get_channels_for_connector(?ROOT_KEY_ACTIONS, ConnectorId), + Sources = get_channels_for_connector(?ROOT_KEY_SOURCES, ConnectorId), + Actions ++ Sources. + +get_channels_for_connector(SourcesOrActions, ConnectorId) -> try emqx_connector_resource:parse_connector_id(ConnectorId) of {ConnectorType, ConnectorName} -> - RootConf = maps:keys(emqx:get_config([?ROOT_KEY], #{})), + RootConf = maps:keys(emqx:get_config([SourcesOrActions], #{})), RelevantBridgeV2Types = [ Type || Type <- RootConf, connector_type(Type) =:= ConnectorType ], lists:flatten([ - get_channels_for_connector(ConnectorName, BridgeV2Type) + get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) || BridgeV2Type <- RelevantBridgeV2Types ]) catch @@ -819,33 +903,55 @@ get_channels_for_connector(ConnectorId) -> [] end. -get_channels_for_connector(ConnectorName, BridgeV2Type) -> - BridgeV2s = emqx:get_config([?ROOT_KEY, BridgeV2Type], #{}), +get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) -> + BridgeV2s = emqx:get_config([SourcesOrActions, BridgeV2Type], #{}), [ - {id(BridgeV2Type, Name, ConnectorName), Conf#{ - bridge_name => bin(Name), - bridge_type => bin(BridgeV2Type) - }} + { + id_with_root_name(SourcesOrActions, BridgeV2Type, Name, ConnectorName), + augment_channel_config(SourcesOrActions, BridgeV2Type, Name, Conf) + } || {Name, Conf} <- maps:to_list(BridgeV2s), bin(ConnectorName) =:= maps:get(connector, Conf, no_name) ]. %%==================================================================== -%% Exported for tests +%% ID related functions %%==================================================================== id(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of - #{connector := ConnectorName} -> - id(BridgeType, BridgeName, ConnectorName); - {error, Reason} -> - throw(Reason) - end. + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). id(BridgeType, BridgeName, ConnectorName) -> + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, ConnectorName). + +id_with_root_name(RootName, BridgeType, BridgeName) -> + case lookup_conf(RootName, BridgeType, BridgeName) of + #{connector := ConnectorName} -> + id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName); + {error, Reason} -> + throw( + {action_source_not_found, #{ + reason => Reason, + root_name => RootName, + type => BridgeType, + name => BridgeName + }} + ) + end. + +id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName) -> ConnectorType = bin(connector_type(BridgeType)), - <<"action:", (bin(BridgeType))/binary, ":", (bin(BridgeName))/binary, ":connector:", - (bin(ConnectorType))/binary, ":", (bin(ConnectorName))/binary>>. + << + (bin(RootName))/binary, + ":", + (bin(BridgeType))/binary, + ":", + (bin(BridgeName))/binary, + ":connector:", + (bin(ConnectorType))/binary, + ":", + (bin(ConnectorName))/binary + >>. connector_type(Type) -> %% remote call so it can be mocked @@ -860,76 +966,65 @@ bridge_v2_type_to_connector_type(Type) -> import_config(RawConf) -> %% actions structure - emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY, config_key_path()). + emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY_ACTIONS, config_key_path()). %%==================================================================== %% Config Update Handler API %%==================================================================== -config_key_path() -> [?ROOT_KEY]. +config_key_path() -> + [?ROOT_KEY_ACTIONS]. -config_key_path_leaf() -> [?ROOT_KEY, '?', '?']. +config_key_path_leaf() -> + [?ROOT_KEY_ACTIONS, '?', '?']. + +config_key_path_sources() -> + [?ROOT_KEY_SOURCES]. + +config_key_path_leaf_sources() -> + [?ROOT_KEY_SOURCES, '?', '?']. %% enable or disable action -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, undefined) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, undefined) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {error, bridge_not_found}; -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, OldAction) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, OldAction) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {ok, OldAction#{<<"enable">> => operation_to_enable(Oper)}}; %% Updates a single action from a specific HTTP API. %% If the connector is not found, the update operation fails. -pre_config_update([?ROOT_KEY, Type, Name], Conf = #{}, _OldConf) -> - action_convert_from_connector(Type, Name, Conf); +pre_config_update([ConfRootKey, Type, Name], Conf = #{}, _OldConf) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + convert_from_connector(ConfRootKey, Type, Name, Conf); %% Batch updates actions when importing a configuration or executing a CLI command. %% Update succeeded even if the connector is not found, alarm in post_config_update -pre_config_update([?ROOT_KEY], Conf = #{}, _OldConfs) -> - {ok, actions_convert_from_connectors(Conf)}. +pre_config_update([ConfRootKey], Conf = #{}, _OldConfs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + {ok, convert_from_connectors(ConfRootKey, Conf)}. -%% Don't crash event the bridge is not found -post_config_update([?ROOT_KEY, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> - AllBridges = emqx:get_config([?ROOT_KEY]), - case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of - undefined -> - ok; - Action -> - ok = uninstall_bridge_v2(Type, Name, Action), - Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), - reload_message_publish_hook(Bridges) - end, - ?tp(bridge_post_config_update_done, #{}), - ok; -%% Create a single bridge failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) -> - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; -%% update bridges failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) -> - ok = uninstall_bridge_v2(BridgeType, BridgeName, OldConf), - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; %% This top level handler will be triggered when the actions path is updated %% with calls to emqx_conf:update([actions], BridgesConf, #{}). -%% such as import_config/1 -%% Notice ** do succeeded even if the connector is not found ** -%% Install a non-exist connector will alarm & log(warn) in install_bridge_v2. -post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> +post_config_update([ConfRootKey], _Req, NewConf, OldConf, _AppEnv) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), %% The config update will be failed if any task in `perform_bridge_changes` failed. - RemoveFun = fun uninstall_bridge_v2/3, - CreateFun = fun install_bridge_v2/3, + RemoveFun = fun(Type, Name, Conf) -> + uninstall_bridge_v2(ConfRootKey, Type, Name, Conf) + end, + CreateFun = fun(Type, Name, Conf) -> + install_bridge_v2(ConfRootKey, Type, Name, Conf) + end, UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) -> - uninstall_bridge_v2(Type, Name, OldBridgeConf), - install_bridge_v2(Type, Name, Conf) + uninstall_bridge_v2(ConfRootKey, Type, Name, OldBridgeConf), + install_bridge_v2(ConfRootKey, Type, Name, Conf) end, Result = perform_bridge_changes([ #{action => RemoveFun, data => Removed}, @@ -942,7 +1037,45 @@ post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> ]), reload_message_publish_hook(NewConf), ?tp(bridge_post_config_update_done, #{}), - Result. + Result; +%% Don't crash even when the bridge is not found +post_config_update([ConfRootKey, Type, Name], '$remove', _, _OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + AllBridges = emqx:get_config([ConfRootKey]), + case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of + undefined -> + ok; + Action -> + ok = uninstall_bridge_v2(ConfRootKey, Type, Name, Action), + Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), + reload_message_publish_hook(Bridges) + end, + ?tp(bridge_post_config_update_done, #{}), + ok; +%% Create a single bridge fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok; +%% update bridges fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = uninstall_bridge_v2(ConfRootKey, BridgeType, BridgeName, OldConf), + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok. diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( @@ -1051,12 +1184,33 @@ is_bridge_v2_type(Type) -> emqx_action_info:is_action_type(Type). bridge_v1_list_and_transform() -> - Bridges = list_with_lookup_fun(fun bridge_v1_lookup_and_transform/2), - [B || B <- Bridges, B =/= not_bridge_v1_compatible_error()]. + BridgesFromActions0 = list_with_lookup_fun( + ?ROOT_KEY_ACTIONS, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromActions1 = [ + B + || B <- BridgesFromActions0, + B =/= not_bridge_v1_compatible_error() + ], + FromActionsNames = maps:from_keys([Name || #{name := Name} <- BridgesFromActions1], true), + BridgesFromSources0 = list_with_lookup_fun( + ?ROOT_KEY_SOURCES, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromSources1 = [ + B + || #{name := SourceBridgeName} = B <- BridgesFromSources0, + B =/= not_bridge_v1_compatible_error(), + %% Action is only shown in case of name conflict + not maps:is_key(SourceBridgeName, FromActionsNames) + ], + BridgesFromActions1 ++ BridgesFromSources1. bridge_v1_lookup_and_transform(ActionType, Name) -> - case lookup(ActionType, Name) of - {ok, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> + case lookup_actions_or_sources(ActionType, Name) of + {ok, ConfRootName, + #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), HasBridgeV1Equivalent = has_bridge_v1_equivalent(ActionType), case HasBridgeV1Equivalent andalso ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of @@ -1065,6 +1219,7 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( + ConfRootName, BridgeV1Type, Name, ActionType, @@ -1082,6 +1237,19 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> Error end. +lookup_actions_or_sources(ActionType, Name) -> + case lookup(?ROOT_KEY_ACTIONS, ActionType, Name) of + {error, not_found} -> + case lookup(?ROOT_KEY_SOURCES, ActionType, Name) of + {ok, SourceInfo} -> + {ok, ?ROOT_KEY_SOURCES, SourceInfo}; + Error -> + Error + end; + {ok, ActionInfo} -> + {ok, ?ROOT_KEY_ACTIONS, ActionInfo} + end. + not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. @@ -1094,18 +1262,18 @@ has_bridge_v1_equivalent(ActionType) -> connector_raw_config(Connector, ConnectorType) -> get_raw_with_defaults(Connector, ConnectorType, <<"connectors">>, emqx_connector_schema). -action_raw_config(Action, ActionType) -> - get_raw_with_defaults(Action, ActionType, <<"actions">>, emqx_bridge_v2_schema). +action_raw_config(ConfRootName, Action, ActionType) -> + get_raw_with_defaults(Action, ActionType, bin(ConfRootName), emqx_bridge_v2_schema). get_raw_with_defaults(Config, Type, TopLevelConf, SchemaModule) -> RawConfig = maps:get(raw_config, Config), fill_defaults(Type, RawConfig, TopLevelConf, SchemaModule). bridge_v1_lookup_and_transform_helper( - BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector + ConfRootName, BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> ConnectorRawConfig = connector_raw_config(Connector, ConnectorType), - ActionRawConfig = action_raw_config(Action, ActionType), + ActionRawConfig = action_raw_config(ConfRootName, Action, ActionType), BridgeV1Config = emqx_action_info:connector_action_config_to_bridge_v1_config( BridgeV1Type, ConnectorRawConfig, ActionRawConfig ), @@ -1136,7 +1304,52 @@ bridge_v1_lookup_and_transform_helper( end. lookup_conf(Type, Name) -> - case emqx:get_config([?ROOT_KEY, Type, Name], not_found) of + lookup_conf(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_conf_if_one_of_sources_actions(Type, Name) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, Type, Name), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, Type, Name), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + {error, bridge_not_found}; + {{error, bridge_not_found}, Conf} -> + Conf; + {Conf, {error, bridge_not_found}} -> + Conf; + {_Conf1, _Conf2} -> + {error, name_conflict_sources_actions} + end. + +is_only_source(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + false; + {{error, bridge_not_found}, _Conf} -> + true; + {_Conf, {error, bridge_not_found}} -> + false; + {_Conf1, _Conf2} -> + false + end. + +get_conf_root_key_if_only_one(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + error({action_or_soruces_not_found, BridgeType, BridgeName}); + {{error, bridge_not_found}, _Conf} -> + ?ROOT_KEY_SOURCES; + {_Conf, {error, bridge_not_found}} -> + ?ROOT_KEY_ACTIONS; + {_Conf1, _Conf2} -> + error({name_clash_action_soruces, BridgeType, BridgeName}) + end. + +lookup_conf(RootName, Type, Name) -> + case emqx:get_config([RootName, Type, Name], not_found) of not_found -> {error, bridge_not_found}; Config -> @@ -1146,7 +1359,7 @@ lookup_conf(Type, Name) -> bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), %% Check if the bridge v2 exists - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, @@ -1158,8 +1371,9 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> true -> %% Using remove + create as update, hence do not delete deps. RemoveDeps = [], + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, BridgeName), PreviousRawConf = emqx:get_raw_config( - [?ROOT_KEY, BridgeV2Type, BridgeName], undefined + [ConfRootKey, BridgeV2Type, BridgeName], undefined ), %% To avoid losing configurations. We have to make sure that no crash occurs %% during deletion and creation of configurations. @@ -1185,17 +1399,18 @@ split_bridge_v1_config_and_create_helper( connector_conf := NewConnectorRawConf, bridge_v2_type := BridgeType, bridge_v2_name := BridgeName, - bridge_v2_conf := NewBridgeV2RawConf + bridge_v2_conf := NewBridgeV2RawConf, + conf_root_key := ConfRootName } = split_and_validate_bridge_v1_config( BridgeV1Type, BridgeName, RawConf, PreviousRawConf ), - _ = PreCreateFun(), do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1210,6 +1425,7 @@ split_bridge_v1_config_and_create_helper( end. do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1220,7 +1436,7 @@ do_connector_and_bridge_create( ) -> case emqx_connector:create(ConnectorType, NewConnectorName, NewConnectorRawConf) of {ok, _} -> - case create(BridgeType, BridgeName, NewBridgeV2RawConf) of + case create(ConfRootName, BridgeType, BridgeName, NewBridgeV2RawConf) of {ok, _} = Result -> Result; {error, Reason1} -> @@ -1257,10 +1473,15 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR } } }, + ConfRootKeyPrevRawConf = + case PreviousRawConf =/= undefined of + true -> get_conf_root_key_if_only_one(BridgeV2Type, BridgeName); + false -> not_used + end, FakeGlobalConfig = emqx_utils_maps:put_if( FakeGlobalConfig0, - bin(?ROOT_KEY), + bin(ConfRootKeyPrevRawConf), #{bin(BridgeV2Type) => #{bin(BridgeName) => PreviousRawConf}}, PreviousRawConf =/= undefined ), @@ -1269,10 +1490,11 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR Output = emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2( FakeGlobalConfig ), + ConfRootKey = get_conf_root_key(Output), NewBridgeV2RawConf = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName) ], @@ -1280,7 +1502,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR ), ConnectorName = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName), <<"connector">> @@ -1303,7 +1525,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR bin(ConnectorName) => NewConnectorRawConf } }, - <<"actions">> => #{ + ConfRootKey => #{ bin(BridgeV2Type) => #{ bin(BridgeName) => NewBridgeV2RawConf } @@ -1323,7 +1545,8 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR connector_conf => NewConnectorRawConf, bridge_v2_type => BridgeV2Type, bridge_v2_name => BridgeName, - bridge_v2_conf => NewBridgeV2RawConf + bridge_v2_conf => NewBridgeV2RawConf, + conf_root_key => ConfRootKey } catch %% validation errors @@ -1331,6 +1554,13 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR {error, Reason1} end. +get_conf_root_key(#{<<"actions">> := _}) -> + <<"actions">>; +get_conf_root_key(#{<<"sources">> := _}) -> + <<"sources">>; +get_conf_root_key(_NoMatch) -> + error({incompatible_bridge_v1, no_action_or_source}). + bridge_v1_create_dry_run(BridgeType, RawConfig0) -> RawConf = maps:without([<<"name">>], RawConfig0), TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), @@ -1356,7 +1586,7 @@ bridge_v1_remove(BridgeV1Type, BridgeName) -> bridge_v1_remove( ActionType, BridgeName, - lookup_conf(ActionType, BridgeName) + lookup_conf_if_one_of_sources_actions(ActionType, BridgeName) ). bridge_v1_remove( @@ -1364,7 +1594,8 @@ bridge_v1_remove( Name, #{connector := ConnectorName} ) -> - case remove(ActionType, Name) of + ConfRootKey = get_conf_root_key_if_only_one(ActionType, Name), + case remove(ConfRootKey, ActionType, Name) of ok -> ConnectorType = connector_type(ActionType), emqx_connector:remove(ConnectorType, ConnectorName); @@ -1384,7 +1615,7 @@ bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) -> BridgeV2Type, BridgeName, RemoveDeps, - lookup_conf(BridgeV2Type, BridgeName) + lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) ). %% Bridge v1 delegated-removal in 3 steps: @@ -1398,9 +1629,10 @@ bridge_v1_check_deps_and_remove( #{connector := ConnectorName} ) -> RemoveConnector = lists:member(connector, RemoveDeps), - case emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of + case maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of ok -> - case remove(BridgeType, BridgeName) of + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + case remove(ConfRootKey, BridgeType, BridgeName) of ok when RemoveConnector -> maybe_delete_channels(BridgeType, BridgeName, ConnectorName); ok -> @@ -1415,6 +1647,14 @@ bridge_v1_check_deps_and_remove(_BridgeType, _BridgeName, _RemoveDeps, Error) -> %% TODO: the connector is gone, for whatever reason, maybe call remove/2 anyway? Error. +maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) -> + case is_only_source(BridgeType, BridgeName) of + true -> + ok; + false -> + emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) + end. + maybe_delete_channels(BridgeType, BridgeName, ConnectorName) -> case connector_has_channels(BridgeType, ConnectorName) of true -> @@ -1467,23 +1707,25 @@ bridge_v1_enable_disable(Action, BridgeType, BridgeName) -> Action, BridgeType, BridgeName, - lookup_conf(BridgeType, BridgeName) + lookup_conf_if_one_of_sources_actions(BridgeType, BridgeName) ); false -> {error, not_bridge_v1_compatible} end. -bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, bridge_not_found}) -> - {error, bridge_not_found}; +bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, Reason}) -> + {error, Reason}; bridge_v1_enable_disable_helper(enable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), {ok, _} = emqx_connector:disable_enable(enable, ConnectorType, ConnectorName), - emqx_bridge_v2:disable_enable(enable, BridgeV2Type, BridgeName); + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_bridge_v2:disable_enable(ConfRootKey, enable, BridgeV2Type, BridgeName); bridge_v1_enable_disable_helper(disable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), - {ok, _} = emqx_bridge_v2:disable_enable(disable, BridgeV2Type, BridgeName), + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + {ok, _} = emqx_bridge_v2:disable_enable(ConfRootKey, disable, BridgeV2Type, BridgeName), emqx_connector:disable_enable(disable, ConnectorType, ConnectorName). bridge_v1_restart(BridgeV1Type, Name) -> @@ -1508,10 +1750,12 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), case emqx_bridge_v2:bridge_v1_is_valid(BridgeV1Type, Name) of true -> + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, Name), connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf_if_one_of_sources_actions(BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ); @@ -1559,12 +1803,12 @@ referenced_connectors_exist(BridgeType, ConnectorNameBin, BridgeName) -> ok end. -actions_convert_from_connectors(Conf) -> +convert_from_connectors(ConfRootKey, Conf) -> maps:map( fun(ActionType, Actions) -> maps:map( fun(ActionName, Action) -> - case action_convert_from_connector(ActionType, ActionName, Action) of + case convert_from_connector(ConfRootKey, ActionType, ActionName, Action) of {ok, NewAction} -> NewAction; {error, _} -> Action end @@ -1575,7 +1819,7 @@ actions_convert_from_connectors(Conf) -> Conf ). -action_convert_from_connector(Type, Name, Action = #{<<"connector">> := ConnectorName}) -> +convert_from_connector(ConfRootKey, Type, Name, Action = #{<<"connector">> := ConnectorName}) -> case get_connector_info(ConnectorName, Type) of {ok, Connector} -> Action1 = emqx_action_info:action_convert_from_connector(Type, Connector, Action), @@ -1585,7 +1829,8 @@ action_convert_from_connector(Type, Name, Action = #{<<"connector">> := Connecto bridge_name => Name, reason => <<"connector_not_found_or_wrong_type">>, bridge_type => Type, - connector_name => ConnectorName + connector_name => ConnectorName, + conf_root_key => ConfRootKey }} end. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index e144f332d..28017f814 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -79,7 +79,7 @@ api_schema(Method) -> hoconsc:union(bridge_api_union(APISchemas)). registered_api_schemas(Method) -> - RegisteredSchemas = emqx_action_info:registered_schema_modules(), + RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_bridge_v2") || {BridgeV2Type, SchemaModule} <- RegisteredSchemas @@ -189,29 +189,43 @@ tags() -> -dialyzer({nowarn_function, roots/0}). roots() -> - case fields(actions) of - [] -> - [ - {actions, - ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} - ]; - _ -> - [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] - end. + ActionsRoot = + case fields(actions) of + [] -> + [ + {actions, + ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} + ]; + _ -> + [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] + end, + SourcesRoot = + [{sources, ?HOCON(?R_REF(sources), #{importance => ?IMPORTANCE_LOW})}], + ActionsRoot ++ SourcesRoot. fields(actions) -> - registered_schema_fields(); + registered_schema_fields_actions(); +fields(sources) -> + registered_schema_fields_sources(); fields(resource_opts) -> resource_opts_fields(_Overrides = []). -registered_schema_fields() -> +registered_schema_fields_actions() -> [ Module:fields(action) - || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules() + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_actions() + ]. + +registered_schema_fields_sources() -> + [ + Module:fields(source) + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_sources() ]. desc(actions) -> ?DESC("desc_bridges_v2"); +desc(sources) -> + ?DESC("desc_sources"); desc(resource_opts) -> ?DESC(emqx_resource_schema, "resource_opts"); desc(_) -> @@ -264,7 +278,7 @@ examples(Method) -> ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), lists:foldl(MergeFun, Examples, ConnectorExamples) end, - SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules()], + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], lists:foldl(Fun, #{}, SchemaModules). top_level_common_action_keys() -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index cc2296d3c..1bee2c92e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -20,8 +20,13 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). +-behaviour(ecpool_worker). + +%% ecpool +-export([connect/1]). -export([on_message_received/3]). +-export([handle_disconnect/1]). %% callbacks of behaviour emqx_resource -export([ @@ -30,11 +35,25 @@ on_stop/2, on_query/3, on_query_async/4, - on_get_status/2 + on_get_status/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channel_status/3, + on_get_channels/1 ]). -export([on_async_result/2]). +-type name() :: term(). + +-type option() :: + {name, name()} + | {ingress, map()} + %% see `emqtt:option()` + | {client_opts, map()}. + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -define(HEALTH_CHECK_TIMEOUT, 1000). -define(INGRESS, "I"). -define(EGRESS, "E"). @@ -42,142 +61,205 @@ %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, HookPoint, ResId) -> + +on_message_received(Msg, HookPoints, ResId) -> emqx_resource_metrics:received_inc(ResId), - emqx_hooks:run(HookPoint, [Msg]). + lists:foreach( + fun(HookPoint) -> + emqx_hooks:run(HookPoint, [Msg]) + end, + HookPoints + ), + ok. %% =================================================================== callback_mode() -> async_if_possible. -on_start(ResourceId, Conf) -> +on_start(ResourceId, #{server := Server} = Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => ResourceId, config => emqx_utils:redact(Conf) }), - case start_ingress(ResourceId, Conf) of + TopicToHandlerIndex = emqx_topic_index:new(), + StartConf = Conf#{topic_to_handler_index => TopicToHandlerIndex}, + case start_mqtt_clients(ResourceId, StartConf) of {ok, Result1} -> - case start_egress(ResourceId, Conf) of - {ok, Result2} -> - {ok, maps:merge(Result1, Result2)}; - {error, Reason} -> - _ = stop_ingress(Result1), - {error, Reason} - end; + {ok, Result1#{ + installed_channels => #{}, + clean_start => maps:get(clean_start, Conf), + topic_to_handler_index => TopicToHandlerIndex, + server => Server + }}; {error, Reason} -> {error, Reason} end. -start_ingress(ResourceId, Conf) -> - ClientOpts = mk_client_opts(ResourceId, ?INGRESS, Conf), - case mk_ingress_config(ResourceId, Conf) of - Ingress = #{} -> - start_ingress(ResourceId, Ingress, ClientOpts); - undefined -> - {ok, #{}} - end. - -start_ingress(ResourceId, Ingress, ClientOpts) -> - PoolName = <>, - PoolSize = choose_ingress_pool_size(ResourceId, Ingress), - Options = [ - {name, PoolName}, - {pool_size, PoolSize}, - {ingress, Ingress}, - {client_opts, ClientOpts} - ], - ok = emqx_resource:allocate_resource(ResourceId, ingress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_ingress, Options) of - ok -> - {ok, #{ingress_pool_name => PoolName}}; - {error, {start_pool_failed, _, Reason}} -> - {error, Reason} - end. - -choose_ingress_pool_size(<>, _) -> - 1; -choose_ingress_pool_size( - ResourceId, - #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} +on_add_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + clean_start := CleanStart + } = OldState, + ChannelId, + #{config_root := actions} = ChannelConfig ) -> - case emqx_topic:parse(RemoteTopic) of - {#share{} = _Filter, _SubOpts} -> - % NOTE: this is shared subscription, many workers may subscribe - PoolSize; - {_Filter, #{}} when PoolSize > 1 -> - % NOTE: this is regular subscription, only one worker should subscribe + %% Publisher channel + %% make a warning if clean_start is set to false + case CleanStart of + false -> + ?tp( + mqtt_clean_start_egress_action_warning, + #{ + channel_id => ChannelId, + resource_id => _InstId + } + ), ?SLOG(warning, #{ - msg => "mqtt_bridge_ingress_pool_size_ignored", - connector => ResourceId, - reason => - "Remote topic filter is not a shared subscription, " - "ingress pool will start with a single worker", - config_pool_size => PoolSize, - pool_size => 1 - }), - 1; - {_Filter, #{}} when PoolSize == 1 -> - 1 - end. + msg => "mqtt_publisher_clean_start_false", + reason => "clean_start is set to false when using MQTT publisher action, " ++ + "which may cause unexpected behavior. " ++ + "For example, if the client ID is already subscribed to topics, " ++ + "we might receive messages that are unhanded.", + channel => ChannelId, + config => emqx_utils:redact(ChannelConfig) + }); + true -> + ok + end, + ChannelState0 = maps:get(parameters, ChannelConfig), + ChannelState = emqx_bridge_mqtt_egress:config(ChannelState0), + NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}; +on_add_channel( + _ResourceId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex, + server := Server + } = OldState, + ChannelId, + #{hookpoints := HookPoints} = ChannelConfig +) -> + %% Add ingress channel + ChannelState0 = maps:get(parameters, ChannelConfig), + ChannelState1 = ChannelState0#{ + hookpoints => HookPoints, + server => Server, + config_root => sources + }, + ChannelState2 = mk_ingress_config(ChannelId, ChannelState1, TopicToHandlerIndex), + ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState2), + NewInstalledChannels = maps:put(ChannelId, ChannelState2, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Conf) -> - % NOTE - % We are ignoring the user configuration here because there's currently no reliable way - % to ensure proper session recovery according to the MQTT spec. - ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, ?EGRESS, Conf)), - case mk_egress_config(Conf) of - Egress = #{} -> - start_egress(ResourceId, Egress, ClientOpts); - undefined -> - {ok, #{}} - end. +on_remove_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex + } = OldState, + ChannelId +) -> + ChannelState = maps:get(ChannelId, InstalledChannels), + case ChannelState of + #{ + config_root := sources + } -> + emqx_bridge_mqtt_ingress:unsubscribe_channel( + PoolName, ChannelState, ChannelId, TopicToHandlerIndex + ), + ok; + _ -> + ok + end, + NewInstalledChannels = maps:remove(ChannelId, InstalledChannels), + %% Update state + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Egress, ClientOpts) -> - PoolName = <>, - PoolSize = maps:get(pool_size, Egress), +on_get_channel_status( + _ResId, + ChannelId, + #{ + installed_channels := Channels + } = _State +) when is_map_key(ChannelId, Channels) -> + %% The channel should be ok as long as the MQTT client is ok + connected. + +on_get_channels(ResId) -> + emqx_bridge_v2:get_channels_for_connector(ResId). + +start_mqtt_clients(ResourceId, Conf) -> + ClientOpts = mk_client_opts(ResourceId, Conf), + start_mqtt_clients(ResourceId, Conf, ClientOpts). + +start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> + PoolName = <>, + #{ + pool_size := PoolSize + } = StartConf, Options = [ {name, PoolName}, {pool_size, PoolSize}, {client_opts, ClientOpts} ], - ok = emqx_resource:allocate_resource(ResourceId, egress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_egress, Options) of + ok = emqx_resource:allocate_resource(ResourceId, pool_name, PoolName), + case emqx_resource_pool:start(PoolName, ?MODULE, Options) of ok -> - {ok, #{ - egress_pool_name => PoolName, - egress_config => emqx_bridge_mqtt_egress:config(Egress) - }}; + {ok, #{pool_name => PoolName}}; {error, {start_pool_failed, _, Reason}} -> {error, Reason} end. -on_stop(ResourceId, _State) -> +on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", connector => ResourceId }), + %% on_stop can be called with State = undefined + StateMap = + case State of + Map when is_map(State) -> + Map; + _ -> + #{} + end, + case maps:get(topic_to_handler_index, StateMap, undefined) of + undefined -> + ok; + TopicToHandlerIndex -> + emqx_topic_index:delete(TopicToHandlerIndex) + end, Allocated = emqx_resource:get_allocated_resources(ResourceId), - ok = stop_ingress(Allocated), - ok = stop_egress(Allocated). + ok = stop_helper(Allocated). -stop_ingress(#{ingress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_ingress(#{}) -> - ok. - -stop_egress(#{egress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_egress(#{}) -> - ok. +stop_helper(#{pool_name := PoolName}) -> + emqx_resource_pool:stop(PoolName). on_query( ResourceId, - {send_message, Msg}, - #{egress_pool_name := PoolName, egress_config := Config} + {ChannelId, Msg}, + #{pool_name := PoolName} = State ) -> - ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - handle_send_result(with_egress_client(PoolName, send, [Msg, Config])); -on_query(ResourceId, {send_message, Msg}, #{}) -> + ?TRACE( + "QUERY", + "send_msg_to_remote_node", + #{ + message => Msg, + connector => ResourceId, + channel_id => ChannelId + } + ), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + handle_send_result(with_egress_client(PoolName, send, [Msg, ChannelConfig])); +on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -187,13 +269,15 @@ on_query(ResourceId, {send_message, Msg}, #{}) -> on_query_async( ResourceId, - {send_message, Msg}, + {ChannelId, Msg}, CallbackIn, - #{egress_pool_name := PoolName, egress_config := Config} + #{pool_name := PoolName} = State ) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - Result = with_egress_client(PoolName, send_async, [Msg, Callback, Config]), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + Result = with_egress_client(PoolName, send_async, [Msg, Callback, ChannelConfig]), case Result of ok -> ok; @@ -202,7 +286,7 @@ on_query_async( {error, Reason} -> {error, classify_error(Reason)} end; -on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> +on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -251,7 +335,7 @@ classify_error(Reason) -> {unrecoverable_error, Reason}. on_get_status(_ResourceId, State) -> - Pools = maps:to_list(maps:with([ingress_pool_name, egress_pool_name], State)), + Pools = maps:to_list(maps:with([pool_name], State)), Workers = [{Pool, Worker} || {Pool, PN} <- Pools, {_Name, Worker} <- ecpool:workers(PN)], try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of Statuses -> @@ -261,12 +345,10 @@ on_get_status(_ResourceId, State) -> connecting end. -get_status({Pool, Worker}) -> +get_status({_Pool, Worker}) -> case ecpool_worker:client(Worker) of - {ok, Client} when Pool == ingress_pool_name -> + {ok, Client} -> emqx_bridge_mqtt_ingress:status(Client); - {ok, Client} when Pool == egress_pool_name -> - emqx_bridge_mqtt_egress:status(Client); {error, _} -> disconnected end. @@ -284,30 +366,19 @@ combine_status(Statuses) -> end. mk_ingress_config( - ResourceId, - #{ - ingress := Ingress = #{remote := _}, - server := Server, - hookpoint := HookPoint - } + ChannelId, + IngressChannelConfig, + TopicToHandlerIndex ) -> - Ingress#{ - server => Server, - on_message_received => {?MODULE, on_message_received, [HookPoint, ResourceId]} - }; -mk_ingress_config(ResourceId, #{ingress := #{remote := _}} = Conf) -> - error({no_hookpoint_provided, ResourceId, Conf}); -mk_ingress_config(_ResourceId, #{}) -> - undefined. - -mk_egress_config(#{egress := Egress = #{remote := _}}) -> - Egress; -mk_egress_config(#{}) -> - undefined. + HookPoints = maps:get(hookpoints, IngressChannelConfig, []), + NewConf = IngressChannelConfig#{ + on_message_received => {?MODULE, on_message_received, [HookPoints, ChannelId]}, + ingress_list => [IngressChannelConfig] + }, + emqx_bridge_mqtt_ingress:config(NewConf, ChannelId, TopicToHandlerIndex). mk_client_opts( ResourceId, - ClientScope, Config = #{ server := Server, keepalive := KeepAlive, @@ -327,14 +398,15 @@ mk_client_opts( % A load balancing server (such as haproxy) is often set up before the emqx broker server. % When the load balancing server enables mqtt connection packet inspection, % non-standard mqtt connection packets might be filtered out by LB. - bridge_mode + bridge_mode, + topic_to_handler_index ], Config ), Name = parse_id_to_name(ResourceId), mk_client_opt_password(Options#{ hosts => [HostPort], - clientid => clientid(Name, ClientScope, Config), + clientid => clientid(Name, Config), connect_timeout => 30, keepalive => ms_to_s(KeepAlive), force_ping => true, @@ -357,9 +429,75 @@ mk_client_opt_password(Options) -> ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Name, ClientScope, _Conf = #{clientid_prefix := Prefix}) when +clientid(Name, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) andalso Prefix =/= <<>> -> - emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name, ClientScope]); -clientid(Name, ClientScope, _Conf) -> - emqx_bridge_mqtt_lib:clientid_base([Name, ClientScope]). + emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name]); +clientid(Name, _Conf) -> + emqx_bridge_mqtt_lib:clientid_base([Name]). + +%% @doc Start an ingress bridge worker. +-spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(Options) -> + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ?SLOG(debug, #{ + msg => "ingress_client_starting", + options => emqx_utils:redact(Options) + }), + Name = proplists:get_value(name, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ClientOpts = proplists:get_value(client_opts, Options), + case emqtt:start_link(mk_client_opts(Name, WorkerId, ClientOpts)) of + {ok, Pid} -> + connect(Pid, Name); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "client_start_failed", + config => emqx_utils:redact(ClientOpts), + reason => Reason + }), + Error + end. + +mk_client_opts( + Name, + WorkerId, + ClientOpts = #{ + clientid := ClientId, + topic_to_handler_index := TopicToHandlerIndex + } +) -> + ClientOpts#{ + clientid := mk_clientid(WorkerId, ClientId), + msg_handler => mk_client_event_handler(Name, TopicToHandlerIndex) + }. + +mk_clientid(WorkerId, ClientId) -> + iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + +mk_client_event_handler(Name, TopicToHandlerIndex) -> + #{ + publish => {fun emqx_bridge_mqtt_ingress:handle_publish/3, [Name, TopicToHandlerIndex]}, + disconnected => {fun ?MODULE:handle_disconnect/1, []} + }. + +-spec connect(pid(), name()) -> + {ok, pid()} | {error, _Reason}. +connect(Pid, Name) -> + case emqtt:connect(Pid) of + {ok, _Props} -> + {ok, Pid}; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "ingress_client_connect_failed", + reason => Reason, + name => Name + }), + _ = catch emqtt:stop(Pid), + Error + end. + +handle_disconnect(_Reason) -> + ok. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 32f9e9295..e863d2a2e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------- %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +30,10 @@ parse_server/1 ]). +-export([ + connector_examples/1 +]). + -import(emqx_schema, [mk_duration/2]). -import(hoconsc, [mk/2, ref/2]). @@ -61,6 +65,39 @@ fields("config") -> } )} ]; +fields("config_connector") -> + [ + {enable, + mk( + boolean(), + #{ + desc => <<"Enable or disable this connector">>, + default => true + } + )}, + {description, emqx_schema:description_schema()}, + {resource_opts, + mk( + hoconsc:ref(creation_opts), + #{ + required => false, + desc => ?DESC(emqx_resource_schema, "creation_opts") + } + )}, + {pool_size, fun egress_pool_size/1} + % {ingress, + % mk( + % hoconsc:array( + % hoconsc:ref(connector_ingress) + % ), + % #{ + % required => {false, recursively}, + % desc => ?DESC("ingress_desc") + % } + % )} + ] ++ fields("server_configs"); +fields(creation_opts) -> + emqx_connector_schema:resource_opts_fields(); fields("server_configs") -> [ {mode, @@ -131,6 +168,7 @@ fields("server_configs") -> fields("ingress") -> [ {pool_size, fun ingress_pool_size/1}, + %% array {remote, mk( ref(?MODULE, "ingress_remote"), @@ -144,6 +182,22 @@ fields("ingress") -> } )} ]; +fields(connector_ingress) -> + [ + {remote, + mk( + ref(?MODULE, "ingress_remote"), + #{desc => ?DESC("ingress_remote")} + )}, + {local, + mk( + ref(?MODULE, "ingress_local"), + #{ + desc => ?DESC("ingress_local"), + importance => ?IMPORTANCE_HIDDEN + } + )} + ]; fields("ingress_remote") -> [ {topic, @@ -269,7 +323,15 @@ fields("egress_remote") -> desc => ?DESC("payload") } )} - ]. + ]; +fields("get_connector") -> + fields("config_connector"); +fields("post_connector") -> + fields("config_connector"); +fields("put_connector") -> + fields("config_connector"); +fields(What) -> + error({emqx_bridge_mqtt_connector_schema, missing_field_handler, What}). ingress_pool_size(desc) -> ?DESC("ingress_pool_size"); @@ -304,3 +366,6 @@ qos() -> parse_server(Str) -> #{hostname := Host, port := Port} = emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS), {Host, Port}. + +connector_examples(_Method) -> + [#{}]. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 2573cad8b..38bdd9665 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -20,33 +20,16 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). - -export([ config/1, send/3, send_async/4 ]). -%% management APIs --export([ - status/1, - info/1 -]). - --type name() :: term(). -type message() :: emqx_types:message() | map(). -type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. -type remote_message() :: #mqtt_msg{}. --type option() :: - {name, name()} - %% see `emqtt:option()` - | {client_opts, map()}. - -type egress() :: #{ local => #{ topic => emqx_types:topic() @@ -54,51 +37,6 @@ remote := emqx_bridge_mqtt_msg:msgvars() }. -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "egress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(WorkerId, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name); - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "egress_client_start_failed", - config => emqx_utils:redact(ClientOpts), - reason => Reason - }), - Error - end. - -mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}. - -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -connect(Pid, Name) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "egress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error - end. - -%% - -spec config(map()) -> egress(). config(#{remote := RC = #{}} = Conf) -> @@ -137,25 +75,3 @@ to_remote_msg(Msg = #{}, Remote) -> props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }. - -%% - --spec info(pid()) -> - [{atom(), term()}]. -info(Pid) -> - emqtt:info(Pid). - --spec status(pid()) -> - emqx_resource:resource_status(). -status(Pid) -> - try - case proplists:get_value(socket, info(Pid)) of - Socket when Socket /= undefined -> - connected; - undefined -> - connecting - end - catch - exit:{noproc, _} -> - disconnected - end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index a051ffbd8..d59318a84 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -17,129 +17,188 @@ -module(emqx_bridge_mqtt_ingress). -include_lib("emqx/include/logger.hrl"). - --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). +-include_lib("emqx/include/emqx_mqtt.hrl"). %% management APIs -export([ status/1, - info/1 + info/1, + subscribe_channel/2, + unsubscribe_channel/4, + config/3 ]). --export([handle_publish/5]). --export([handle_disconnect/1]). +-export([handle_publish/3]). --type name() :: term(). +subscribe_channel(PoolName, ChannelConfig) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + Results = [ + subscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize) + || {{Name, Idx}, Pid} <- Workers + ], + case proplists:get_value(error, Results, ok) of + ok -> + ok; + Error -> + Error + end. --type option() :: - {name, name()} - | {ingress, map()} - %% see `emqtt:option()` - | {client_opts, map()}. +subscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize); + {error, Reason} -> + error({client_not_found, Reason}) + end. --type ingress() :: #{ - server := string(), - remote := #{ - topic := emqx_types:topic(), - qos => emqx_types:qos() - }, - local := emqx_bridge_mqtt_msg:msgvars(), - on_message_received := {module(), atom(), [term()]} -}. - -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "ingress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - Ingress = config(proplists:get_value(ingress, Options), Name), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(Name, WorkerId, Ingress, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name, Ingress); +subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize) -> + IngressList = maps:get(ingress_list, Ingress, []), + SubscribeResults = subscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name + ), + %% Find error if any using proplists:get_value/2 + case proplists:get_value(error, SubscribeResults, ok) of + ok -> + ok; {error, Reason} = Error -> ?SLOG(error, #{ - msg => "client_start_failed", - config => emqx_utils:redact(ClientOpts), + msg => "ingress_client_subscribe_failed", + ingress => Ingress, + name => Name, reason => Reason }), Error end. -mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{ - clientid := mk_clientid(WorkerId, ClientId), - msg_handler => mk_client_event_handler(Name, Ingress) - }. +subscribe_remote_topics(Pid, IngressList, WorkerIdx, PoolSize, Name) -> + [subscribe_remote_topic(Pid, Ingress, WorkerIdx, PoolSize, Name) || Ingress <- IngressList]. -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -mk_client_event_handler(Name, Ingress = #{}) -> - IngressVars = maps:with([server], Ingress), - OnMessage = maps:get(on_message_received, Ingress, undefined), - LocalPublish = - case Ingress of - #{local := Local = #{topic := _}} -> - Local; - #{} -> - undefined - end, - #{ - publish => {fun ?MODULE:handle_publish/5, [Name, OnMessage, LocalPublish, IngressVars]}, - disconnected => {fun ?MODULE:handle_disconnect/1, []} - }. - --spec connect(pid(), name(), ingress()) -> - {ok, pid()} | {error, _Reason}. -connect(Pid, Name, Ingress) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - case subscribe_remote_topic(Pid, Ingress) of - {ok, _, _RCs} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "ingress_client_subscribe_failed", - ingress => Ingress, - name => Name, - reason => Reason - }), - _ = catch emqtt:stop(Pid), - Error - end; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "ingress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error +subscribe_remote_topic( + Pid, #{remote := #{topic := RemoteTopic, qos := QoS}} = _Remote, WorkerIdx, PoolSize, Name +) -> + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _LogWarn = true) of + true -> + emqtt:subscribe(Pid, RemoteTopic, QoS); + false -> + ok end. -subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> - emqtt:subscribe(Pid, RemoteTopic, QoS). +should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, LogWarn) -> + IsFirstWorker = WorkerIdx == 1, + case emqx_topic:parse(RemoteTopic) of + {#share{} = _Filter, _SubOpts} -> + % NOTE: this is shared subscription, many workers may subscribe + true; + {_Filter, #{}} when PoolSize > 1, IsFirstWorker, LogWarn -> + % NOTE: this is regular subscription, only one worker should subscribe + ?SLOG(warning, #{ + msg => "mqtt_pool_size_ignored", + connector => Name, + reason => + "Remote topic filter is not a shared subscription, " + "only a single connection will be used from the connection pool", + config_pool_size => PoolSize, + pool_size => PoolSize + }), + IsFirstWorker; + {_Filter, #{}} -> + % NOTE: this is regular subscription, only one worker should subscribe + IsFirstWorker + end. -%% +unsubscribe_channel(PoolName, ChannelConfig, ChannelId, TopicToHandlerIndex) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + _ = [ + unsubscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize, ChannelId, TopicToHandlerIndex) + || {{Name, Idx}, Pid} <- Workers + ], + ok. --spec config(map(), name()) -> - ingress(). -config(#{remote := RC, local := LC} = Conf, BridgeName) -> - Conf#{ +unsubscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex + ); + {error, Reason} -> + error({client_not_found, Reason}) + end. + +unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex +) -> + IngressList = maps:get(ingress_list, Ingress, []), + unsubscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ). + +unsubscribe_remote_topics( + Pid, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex +) -> + [ + unsubscribe_remote_topic( + Pid, Ingress, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ) + || Ingress <- IngressList + ]. + +unsubscribe_remote_topic( + Pid, + #{remote := #{topic := RemoteTopic}} = _Remote, + WorkerIdx, + PoolSize, + Name, + ChannelId, + TopicToHandlerIndex +) -> + emqx_topic_index:delete(RemoteTopic, ChannelId, TopicToHandlerIndex), + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _NoWarn = false) of + true -> + case emqtt:unsubscribe(Pid, RemoteTopic) of + {ok, _Properties, _ReasonCodes} -> + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "unsubscribe_mqtt_topic_failed", + channel_id => Name, + reason => Reason + }), + ok + end; + false -> + ok + end. + +config(#{ingress_list := IngressList} = Conf, Name, TopicToHandlerIndex) -> + NewIngressList = [ + fix_remote_config(Ingress, Name, TopicToHandlerIndex, Conf) + || Ingress <- IngressList + ], + Conf#{ingress_list => NewIngressList}. + +fix_remote_config(#{remote := RC, local := LC}, BridgeName, TopicToHandlerIndex, Conf) -> + FixedConf = Conf#{ remote => parse_remote(RC, BridgeName), local => emqx_bridge_mqtt_msg:parse(LC) - }. + }, + insert_to_topic_to_handler_index(FixedConf, TopicToHandlerIndex, BridgeName), + FixedConf. -parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> +insert_to_topic_to_handler_index( + #{remote := #{topic := Topic}} = Conf, TopicToHandlerIndex, BridgeName +) -> + TopicPattern = + case emqx_topic:parse(Topic) of + {#share{group = _Group, topic = TP}, _} -> + TP; + _ -> + Topic + end, + emqx_topic_index:insert(TopicPattern, BridgeName, Conf, TopicToHandlerIndex). + +parse_remote(#{qos := QoSIn} = Remote, BridgeName) -> QoS = downgrade_ingress_qos(QoSIn), case QoS of QoSIn -> @@ -152,7 +211,7 @@ parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> name => BridgeName }) end, - Conf#{qos => QoS}. + Remote#{qos => QoS}. downgrade_ingress_qos(2) -> 1; @@ -183,17 +242,39 @@ status(Pid) -> %% -handle_publish(#{properties := Props} = MsgIn, Name, OnMessage, LocalPublish, IngressVars) -> - Msg = import_msg(MsgIn, IngressVars), +handle_publish( + #{properties := Props, topic := Topic} = MsgIn, + Name, + TopicToHandlerIndex +) -> ?SLOG(debug, #{ msg => "ingress_publish_local", - message => Msg, + message => MsgIn, name => Name }), - maybe_on_message_received(Msg, OnMessage), - maybe_publish_local(Msg, LocalPublish, Props). + Matches = emqx_topic_index:matches(Topic, TopicToHandlerIndex, []), + lists:foreach( + fun(Match) -> + handle_match(TopicToHandlerIndex, Match, MsgIn, Name, Props) + end, + Matches + ), + ok. -handle_disconnect(_Reason) -> +handle_match( + TopicToHandlerIndex, + Match, + MsgIn, + _Name, + Props +) -> + [ChannelConfig] = emqx_topic_index:get_record(Match, TopicToHandlerIndex), + #{on_message_received := OnMessage} = ChannelConfig, + Msg = import_msg(MsgIn, ChannelConfig), + + maybe_on_message_received(Msg, OnMessage), + LocalPublish = maps:get(local, ChannelConfig, undefined), + _ = maybe_publish_local(Msg, LocalPublish, Props), ok. maybe_on_message_received(Msg, {Mod, Func, Args}) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl new file mode 100644 index 000000000..6bcdc611b --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -0,0 +1,221 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_pubsub_action_info). + +-behaviour(emqx_action_info). + +-export([ + bridge_v1_type_name/0, + action_type_name/0, + connector_type_name/0, + schema_module/0, + bridge_v1_config_to_connector_config/1, + bridge_v1_config_to_action_config/2, + connector_action_config_to_bridge_v1_config/2, + is_source/0 +]). + +bridge_v1_type_name() -> mqtt. + +action_type_name() -> mqtt. + +connector_type_name() -> mqtt. + +schema_module() -> emqx_bridge_mqtt_pubsub_schema. + +is_source() -> true. + +bridge_v1_config_to_connector_config(Config) -> + %% Transform the egress part to mqtt_publisher connector config + SimplifiedConfig = check_and_simplify_bridge_v1_config(Config), + ConnectorConfigMap = make_connector_config_from_bridge_v1_config(SimplifiedConfig), + {mqtt, ConnectorConfigMap}. + +make_connector_config_from_bridge_v1_config(Config) -> + ConnectorConfigSchema = emqx_bridge_mqtt_connector_schema:fields("config_connector"), + ConnectorTopFields = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ConnectorConfigSchema + ], + ConnectorConfigMap = maps:with(ConnectorTopFields, Config), + ResourceOptsSchema = emqx_bridge_mqtt_connector_schema:fields(creation_opts), + ResourceOptsTopFields = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ResourceOptsSchema + ], + ResourceOptsMap = maps:get(<<"resource_opts">>, ConnectorConfigMap, #{}), + ResourceOptsMap2 = maps:with(ResourceOptsTopFields, ResourceOptsMap), + ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), + IngressMap0 = maps:get(<<"ingress">>, Config, #{}), + EgressMap = maps:get(<<"egress">>, Config, #{}), + % %% Move pool_size to the top level + PoolSizeIngress = maps:get(<<"pool_size">>, IngressMap0, undefined), + PoolSize = + case PoolSizeIngress of + undefined -> + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + maps:get(<<"pool_size">>, EgressMap, DefaultPoolSize); + _ -> + PoolSizeIngress + end, + % IngressMap1 = maps:remove(<<"pool_size">>, IngressMap0), + %% Remove ingress part from the config + ConnectorConfigMap3 = maps:remove(<<"ingress">>, ConnectorConfigMap2), + %% Remove egress part from the config + ConnectorConfigMap4 = maps:remove(<<"egress">>, ConnectorConfigMap3), + ConnectorConfigMap5 = maps:put(<<"pool_size">>, PoolSize, ConnectorConfigMap4), + % ConnectorConfigMap4 = + % case IngressMap1 =:= #{} of + % true -> + % ConnectorConfigMap3; + % false -> + % maps:put(<<"ingress">>, [IngressMap1], ConnectorConfigMap3) + % end, + ConnectorConfigMap5. + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + SimplifiedConfig = check_and_simplify_bridge_v1_config(BridgeV1Config), + bridge_v1_config_to_action_config_helper( + SimplifiedConfig, ConnectorName + ). + +bridge_v1_config_to_action_config_helper( + #{ + <<"egress">> := EgressMap0 + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_publisher_action"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), + LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), + EgressMap1 = maps:remove(<<"local">>, EgressMap0), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), + ConfigMap3 = + case LocalTopic of + undefined -> + ConfigMap2; + _ -> + maps:put(<<"local_topic">>, LocalTopic, ConfigMap2) + end, + {action, mqtt, ConfigMap3}; +bridge_v1_config_to_action_config_helper( + #{ + <<"ingress">> := IngressMap + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + IngressMap1 = maps:remove(<<"pool_size">>, IngressMap), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, IngressMap1, ConfigMap1), + {source, mqtt, ConfigMap2}; +bridge_v1_config_to_action_config_helper( + _Config, + _ConnectorName +) -> + error({incompatible_bridge_v1, no_matching_action_or_source}). + +general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields +) -> + ShemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- SchemaFields + ], + ActionConfig0 = maps:with(ShemaFieldsNames, Config), + ResourceOptsSchemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ResourceOptsSchemaFields + ], + ResourceOptsMap = maps:get(<<"resource_opts">>, ActionConfig0, #{}), + ResourceOptsMap2 = maps:with(ResourceOptsSchemaFieldsNames, ResourceOptsMap), + %% Only put resource_opts if the original config has it + ActionConfig1 = + case maps:is_key(<<"resource_opts">>, ActionConfig0) of + true -> + maps:put(<<"resource_opts">>, ResourceOptsMap2, ActionConfig0); + false -> + ActionConfig0 + end, + ActionConfig2 = maps:put(<<"connector">>, ConnectorName, ActionConfig1), + ActionConfig2. + +check_and_simplify_bridge_v1_config( + #{ + <<"egress">> := EgressMap + } = Config +) when map_size(EgressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"egress">>, Config)); +check_and_simplify_bridge_v1_config( + #{ + <<"ingress">> := IngressMap + } = Config +) when map_size(IngressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"ingress">>, Config)); +check_and_simplify_bridge_v1_config(#{ + <<"egress">> := _EGressMap, + <<"ingress">> := _InGressMap +}) -> + %% We should crash beacuse we don't support upgrading when ingress and egress exist at the same time + error( + {unsupported_config, + <<"Upgrade not supported when ingress and egress exist in the same MQTT bridge. Please divide the egress and ingress part to separate bridges in the configuration.">>} + ); +check_and_simplify_bridge_v1_config(SimplifiedConfig) -> + SimplifiedConfig. + +connector_action_config_to_bridge_v1_config( + ConnectorConfig, ActionConfig +) -> + Params = maps:get(<<"parameters">>, ActionConfig, #{}), + ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), + ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), + ResourceOpts = maps:merge(ResourceOptsConnector, ResourceOptsAction), + %% Check the direction of the action + Direction = + case maps:get(<<"remote">>, Params) of + #{<<"retain">> := _} -> + %% Only source has retain + <<"publisher">>; + _ -> + <<"subscriber">> + end, + Parms2 = maps:remove(<<"direction">>, Params), + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + PoolSize = maps:get(<<"pool_size">>, ConnectorConfig, DefaultPoolSize), + Parms3 = maps:put(<<"pool_size">>, PoolSize, Parms2), + ConnectorConfig2 = maps:remove(<<"pool_size">>, ConnectorConfig), + LocalTopic = maps:get(<<"local_topic">>, ActionConfig, undefined), + BridgeV1Conf0 = + case {Direction, LocalTopic} of + {<<"publisher">>, undefined} -> + #{<<"egress">> => Parms3}; + {<<"publisher">>, LocalT} -> + #{ + <<"egress">> => Parms3, + <<"local">> => + #{ + <<"topic">> => LocalT + } + }; + {<<"subscriber">>, _} -> + #{<<"ingress">> => Parms3} + end, + BridgeV1Conf1 = maps:merge(BridgeV1Conf0, ConnectorConfig2), + BridgeV1Conf2 = BridgeV1Conf1#{ + <<"resource_opts">> => ResourceOpts + }, + BridgeV1Conf2. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl new file mode 100644 index 000000000..2aba6e8ea --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -0,0 +1,129 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_pubsub_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/2]). + +-export([roots/0, fields/1, desc/1, namespace/0]). + +-export([ + bridge_v2_examples/1, + conn_bridge_examples/1 +]). + +%%====================================================================================== +%% Hocon Schema Definitions +namespace() -> "bridge_mqtt_publisher". + +roots() -> []. + +fields(action) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_publisher_action")), + #{ + desc => <<"MQTT Publisher Action Config">>, + required => false + } + )}; +fields("mqtt_publisher_action") -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + hoconsc:ref(?MODULE, action_parameters), + #{ + required => true, + desc => ?DESC("action_parameters") + } + ) + ); +fields(action_parameters) -> + Fields0 = emqx_bridge_mqtt_connector_schema:fields("egress"), + Fields1 = proplists:delete(pool_size, Fields0), + Fields2 = proplists:delete(local, Fields1), + Fields2; +fields(source) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_subscriber_source")), + #{ + desc => <<"MQTT Subscriber Source Config">>, + required => false + } + )}; +fields("mqtt_subscriber_source") -> + emqx_bridge_v2_schema:make_consumer_action_schema( + hoconsc:mk( + hoconsc:ref(?MODULE, ingress_parameters), + #{ + required => true, + desc => ?DESC("source_parameters") + } + ) + ); +fields(ingress_parameters) -> + Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), + Fields1 = proplists:delete(pool_size, Fields0), + Fields1; +fields("resource_opts") -> + UnsupportedOpts = [enable_batch, batch_size, batch_time], + lists:filter( + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, + emqx_resource_schema:fields("creation_opts") + ); +fields("get_connector") -> + emqx_bridge_mqtt_connector_schema:fields("config_connector"); +fields("get_bridge_v2") -> + fields("mqtt_publisher_action"); +fields("post_bridge_v2") -> + fields("mqtt_publisher_action"); +fields("put_bridge_v2") -> + fields("mqtt_publisher_action"); +fields(What) -> + error({emqx_bridge_mqtt_pubsub_schema, missing_field_handler, What}). +%% v2: api schema +%% The parameter equls to +%% `get_bridge_v2`, `post_bridge_v2`, `put_bridge_v2` from emqx_bridge_v2_schema:api_schema/1 +%% `get_connector`, `post_connector`, `put_connector` from emqx_connector_schema:api_schema/1 +%%-------------------------------------------------------------------- +%% v1/v2 + +desc("config") -> + ?DESC("desc_config"); +desc("resource_opts") -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; +desc("config_connector") -> + ?DESC("desc_config"); +desc("http_action") -> + ?DESC("desc_config"); +desc("parameters_opts") -> + ?DESC("config_parameters_opts"); +desc(_) -> + undefined. + +bridge_v2_examples(_Method) -> + [ + #{} + ]. + +conn_bridge_examples(_Method) -> + [ + #{} + ]. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6d1ff0915..bd3fb68de 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -238,7 +238,8 @@ t_conf_bridge_authn_passfile(Config) -> post, uri(["bridges"]), ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ - <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + <<"name">> => <<"t_conf_bridge_authn_no_passfile">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} } ), ?assertMatch({match, _}, re:run(Reason, <<"failed_to_read_secret_file">>)). @@ -397,32 +398,25 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), ok. -t_mqtt_egress_bridge_ignores_clean_start(_) -> +t_mqtt_egress_bridge_warns_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), - BridgeID = create_bridge( - ?SERVER_CONF#{ - <<"name">> => BridgeName, - <<"egress">> => ?EGRESS_CONF, - <<"clean_start">> => false - } - ), + Action = fun() -> + BridgeID = create_bridge( + ?SERVER_CONF#{ + <<"name">> => BridgeName, + <<"egress">> => ?EGRESS_CONF, + <<"clean_start">> => false + } + ), - ResourceID = emqx_bridge_resource:resource_id(BridgeID), - {ok, _Group, #{state := #{egress_pool_name := EgressPoolName}}} = - emqx_resource_manager:lookup_cached(ResourceID), - ClientInfo = ecpool:pick_and_do( - EgressPoolName, - {emqx_bridge_mqtt_egress, info, []}, - no_handover + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []) + end, + ?wait_async_action( + Action(), + #{?snk_kind := mqtt_clean_start_egress_action_warning}, + 10000 ), - ?assertMatch( - #{clean_start := true}, - maps:from_list(ClientInfo) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - ok. t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index a58a1ef3d..f85109080 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -51,11 +51,11 @@ -export([parse_url/1]). --callback connector_config(ParsedConfig) -> +-callback connector_config(ParsedConfig, Context) -> ParsedConfig when - ParsedConfig :: #{atom() => any()}. --optional_callbacks([connector_config/1]). + ParsedConfig :: #{atom() => any()}, Context :: #{atom() => any()}. +-optional_callbacks([connector_config/2]). -if(?EMQX_RELEASE_EDITION == ee). connector_to_resource_type(ConnectorType) -> @@ -81,6 +81,10 @@ connector_impl_module(_ConnectorType) -> connector_to_resource_type_ce(http) -> emqx_bridge_http_connector; +connector_to_resource_type_ce(mqtt) -> + emqx_bridge_mqtt_connector; +% connector_to_resource_type_ce(mqtt_subscriber) -> +% emqx_bridge_mqtt_subscriber_connector; connector_to_resource_type_ce(ConnectorType) -> error({no_bridge_v2, ConnectorType}). @@ -276,6 +280,12 @@ remove(Type, Name, _Conf, _Opts) -> emqx_resource:remove_local(resource_id(Type, Name)). %% convert connector configs to what the connector modules want +parse_confs( + <<"mqtt">> = Type, + Name, + Conf +) -> + insert_hookpoints(Type, Name, Conf); parse_confs( <<"http">>, _Name, @@ -307,6 +317,13 @@ parse_confs( parse_confs(ConnectorType, Name, Config) -> connector_config(ConnectorType, Name, Config). +insert_hookpoints(Type, Name, Conf) -> + BId = emqx_bridge_resource:bridge_id(Type, Name), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + ConnectorHookpoint = connector_hookpoint(BId), + HookPoints = [BridgeHookpoint, ConnectorHookpoint], + Conf#{hookpoints => HookPoints}. + connector_config(ConnectorType, Name, Config) -> Mod = connector_impl_module(ConnectorType), case erlang:function_exported(Mod, connector_config, 2) of diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index b043ebacd..74b92c165 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -90,7 +90,9 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector") + api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector"), + % api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt_subscriber">>, Method ++ "_connector"), + api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector") ]. api_ref(Module, Type, Method) -> @@ -110,10 +112,11 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - [emqx_bridge_http_schema] ++ emqx_connector_ee_schema:schema_modules(). + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ + emqx_connector_ee_schema:schema_modules(). -else. schema_modules() -> - [emqx_bridge_http_schema]. + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema]. -endif. %% @doc Return old bridge(v1) and/or connector(v2) type @@ -136,6 +139,8 @@ connector_type_to_bridge_types(influxdb) -> [influxdb, influxdb_api_v1, influxdb_api_v2]; connector_type_to_bridge_types(mysql) -> [mysql]; +connector_type_to_bridge_types(mqtt) -> + [mqtt]; connector_type_to_bridge_types(pgsql) -> [pgsql]; connector_type_to_bridge_types(redis) -> @@ -151,7 +156,8 @@ connector_type_to_bridge_types(iotdb) -> connector_type_to_bridge_types(elasticsearch) -> [elasticsearch]. -actions_config_name() -> <<"actions">>. +actions_config_name(action) -> <<"actions">>; +actions_config_name(source) -> <<"sources">>. has_connector_field(BridgeConf, ConnectorFields) -> lists:any( @@ -185,40 +191,58 @@ bridge_configs_to_transform( end. split_bridge_to_connector_and_action( - {ConnectorsMap, {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig}} + { + {ConnectorsMap, OrgConnectorType}, + {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig} + } ) -> - ConnectorMap = + {ConnectorMap, ConnectorType} = case emqx_action_info:has_custom_bridge_v1_config_to_connector_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_connector_config( - BridgeType, BridgeV1Conf - ); + case + emqx_action_info:bridge_v1_config_to_connector_config( + BridgeType, BridgeV1Conf + ) + of + {ConType, ConMap} -> + {ConMap, ConType}; + ConMap -> + {ConMap, OrgConnectorType} + end; false -> %% We do an automatic transformation to get the connector config %% if the callback is not defined. %% Get connector fields from bridge config - lists:foldl( - fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> - ConnectorFieldNameBin = to_bin(ConnectorFieldName), - case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of - true -> - PrevFieldConfig = - maybe_project_to_connector_resource_opts( + NewCConMap = + lists:foldl( + fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> + ConnectorFieldNameBin = to_bin(ConnectorFieldName), + case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of + true -> + PrevFieldConfig = + maybe_project_to_connector_resource_opts( + ConnectorFieldNameBin, + maps:get(ConnectorFieldNameBin, BridgeV1Conf) + ), + NewToTransform0 = maps:put( ConnectorFieldNameBin, - maps:get(ConnectorFieldNameBin, BridgeV1Conf) + PrevFieldConfig, + ToTransformSoFar ), - maps:put( - ConnectorFieldNameBin, - PrevFieldConfig, + NewToTransform1 = maps:put( + to_bin(ConnectorFieldName), + maps:get(to_bin(ConnectorFieldName), BridgeV1Conf), + NewToTransform0 + ), + NewToTransform1; + false -> ToTransformSoFar - ); - false -> - ToTransformSoFar - end - end, - #{}, - ConnectorFields - ) + end + end, + #{}, + ConnectorFields + ), + {NewCConMap, OrgConnectorType} end, %% Generate a connector name, if needed. Avoid doing so if there was a previous config. ConnectorName = @@ -226,18 +250,29 @@ split_bridge_to_connector_and_action( #{<<"connector">> := ConnectorName0} -> ConnectorName0; _ -> generate_connector_name(ConnectorsMap, BridgeName, 0) end, - ActionMap = + OrgActionType = emqx_action_info:bridge_v1_type_to_action_type(BridgeType), + {ActionMap, ActionType, ActionOrSource} = case emqx_action_info:has_custom_bridge_v1_config_to_action_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_action_config( - BridgeType, BridgeV1Conf, ConnectorName - ); + case + emqx_action_info:bridge_v1_config_to_action_config( + BridgeType, BridgeV1Conf, ConnectorName + ) + of + {ActionOrSource0, ActionType0, ActionMap0} -> + {ActionMap0, ActionType0, ActionOrSource0}; + ActionMap0 -> + {ActionMap0, OrgActionType, action} + end; false -> - transform_bridge_v1_config_to_action_config( - BridgeV1Conf, ConnectorName, ConnectorFields - ) + ActionMap0 = + transform_bridge_v1_config_to_action_config( + BridgeV1Conf, ConnectorName, ConnectorFields + ), + {ActionMap0, OrgActionType} end, - {BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}. + {BridgeType, BridgeName, ActionMap, ActionType, ActionOrSource, ConnectorName, ConnectorMap, + ConnectorType}. maybe_project_to_connector_resource_opts(<<"resource_opts">>, OldResourceOpts) -> project_to_connector_resource_opts(OldResourceOpts); @@ -307,9 +342,9 @@ generate_connector_name(ConnectorsMap, BridgeName, Attempt) -> ConnectorNameList = case Attempt of 0 -> - io_lib:format("connector_~s", [BridgeName]); + io_lib:format("~s", [BridgeName]); _ -> - io_lib:format("connector_~s_~p", [BridgeName, Attempt + 1]) + io_lib:format("~s_~p", [BridgeName, Attempt + 1]) end, ConnectorName = iolist_to_binary(ConnectorNameList), case maps:is_key(ConnectorName, ConnectorsMap) of @@ -340,7 +375,10 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), ConnectorsWithTypeMap = maps:get(to_bin(ConnectorType), ConnectorsConfMap, #{}), BridgeConfigsToTransformWithConnectorConf = lists:zip( - lists:duplicate(length(BridgeConfigsToTransform), ConnectorsWithTypeMap), + lists:duplicate( + length(BridgeConfigsToTransform), + {ConnectorsWithTypeMap, ConnectorType} + ), BridgeConfigsToTransform ), ActionConnectorTuples = lists:map( @@ -349,10 +387,14 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), %% Add connectors and actions and remove bridges lists:foldl( - fun({BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}, RawConfigSoFar) -> + fun( + {BridgeType, BridgeName, ActionMap, NewActionType, ActionOrSource, ConnectorName, + ConnectorMap, NewConnectorType}, + RawConfigSoFar + ) -> %% Add connector RawConfigSoFar1 = emqx_utils_maps:deep_put( - [<<"connectors">>, to_bin(ConnectorType), ConnectorName], + [<<"connectors">>, to_bin(NewConnectorType), ConnectorName], RawConfigSoFar, ConnectorMap ), @@ -362,12 +404,21 @@ transform_old_style_bridges_to_connector_and_actions_of_type( RawConfigSoFar1 ), %% Add action - ActionType = emqx_action_info:bridge_v1_type_to_action_type(to_bin(BridgeType)), - RawConfigSoFar3 = emqx_utils_maps:deep_put( - [actions_config_name(), to_bin(ActionType), BridgeName], - RawConfigSoFar2, - ActionMap - ), + RawConfigSoFar3 = + case ActionMap of + none -> + RawConfigSoFar2; + _ -> + emqx_utils_maps:deep_put( + [ + actions_config_name(ActionOrSource), + to_bin(NewActionType), + BridgeName + ], + RawConfigSoFar2, + ActionMap + ) + end, RawConfigSoFar3 end, RawConfig, @@ -454,7 +505,23 @@ fields(connectors) -> desc => <<"HTTP Connector Config">>, required => false } + )}, + {mqtt, + mk( + hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + #{ + desc => <<"MQTT Publisher Connector Config">>, + required => false + } )} + % {mqtt_subscriber, + % mk( + % hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + % #{ + % desc => <<"MQTT Subscriber Connector Config">>, + % required => false + % } + % )} ] ++ enterprise_fields_connectors(); fields("node_status") -> [ From 145ed2e6320015025ec5542c4912e8b9a4e6de17 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 15:34:23 +0100 Subject: [PATCH 02/28] fix: elvis style error --- .../src/emqx_bridge_mqtt_pubsub_action_info.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 6bcdc611b..365af1335 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -171,8 +171,11 @@ check_and_simplify_bridge_v1_config(#{ }) -> %% We should crash beacuse we don't support upgrading when ingress and egress exist at the same time error( - {unsupported_config, - <<"Upgrade not supported when ingress and egress exist in the same MQTT bridge. Please divide the egress and ingress part to separate bridges in the configuration.">>} + {unsupported_config, << + "Upgrade not supported when ingress and egress exist in the " + "same MQTT bridge. Please divide the egress and ingress part " + "to separate bridges in the configuration." + >>} ); check_and_simplify_bridge_v1_config(SimplifiedConfig) -> SimplifiedConfig. From 886ed55374a110e7d22e28c8a161b76a44e6304c Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 15:55:38 +0100 Subject: [PATCH 03/28] fix: don't call non-existing function --- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 1bee2c92e..6f9465cb0 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -234,7 +234,7 @@ on_stop(ResourceId, State) -> undefined -> ok; TopicToHandlerIndex -> - emqx_topic_index:delete(TopicToHandlerIndex) + ets:delete(TopicToHandlerIndex) end, Allocated = emqx_resource:get_allocated_resources(ResourceId), ok = stop_helper(Allocated). From 2ecc775fb7202a4145824ba6820bc8b28cae40c9 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 16:04:02 +0100 Subject: [PATCH 04/28] style: remove commented out code and fix copyright headers --- .../emqx_bridge_mqtt_pubsub_action_info.erl | 24 +++++++++++-------- .../src/emqx_bridge_mqtt_schema.erl | 1 + 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 365af1335..0a1eefd82 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_bridge_mqtt_pubsub_action_info). @@ -50,7 +62,7 @@ make_connector_config_from_bridge_v1_config(Config) -> ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), IngressMap0 = maps:get(<<"ingress">>, Config, #{}), EgressMap = maps:get(<<"egress">>, Config, #{}), - % %% Move pool_size to the top level + %% Move pool_size to the top level PoolSizeIngress = maps:get(<<"pool_size">>, IngressMap0, undefined), PoolSize = case PoolSizeIngress of @@ -60,19 +72,11 @@ make_connector_config_from_bridge_v1_config(Config) -> _ -> PoolSizeIngress end, - % IngressMap1 = maps:remove(<<"pool_size">>, IngressMap0), %% Remove ingress part from the config ConnectorConfigMap3 = maps:remove(<<"ingress">>, ConnectorConfigMap2), %% Remove egress part from the config ConnectorConfigMap4 = maps:remove(<<"egress">>, ConnectorConfigMap3), ConnectorConfigMap5 = maps:put(<<"pool_size">>, PoolSize, ConnectorConfigMap4), - % ConnectorConfigMap4 = - % case IngressMap1 =:= #{} of - % true -> - % ConnectorConfigMap3; - % false -> - % maps:put(<<"ingress">>, [IngressMap1], ConnectorConfigMap3) - % end, ConnectorConfigMap5. bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl index a312dfaa9..37d0b4f03 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -13,6 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- + -module(emqx_bridge_mqtt_schema). -include_lib("typerefl/include/types.hrl"). From 139da6d720d0f9941865ea936e2e577951e41bf6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:26:05 -0300 Subject: [PATCH 05/28] fix: don't double-write the transformed config; return a triplet in all cases --- .../src/schema/emqx_connector_schema.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 74b92c165..b51be53ed 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -224,17 +224,11 @@ split_bridge_to_connector_and_action( ConnectorFieldNameBin, maps:get(ConnectorFieldNameBin, BridgeV1Conf) ), - NewToTransform0 = maps:put( + maps:put( ConnectorFieldNameBin, PrevFieldConfig, ToTransformSoFar - ), - NewToTransform1 = maps:put( - to_bin(ConnectorFieldName), - maps:get(to_bin(ConnectorFieldName), BridgeV1Conf), - NewToTransform0 - ), - NewToTransform1; + ); false -> ToTransformSoFar end @@ -269,7 +263,7 @@ split_bridge_to_connector_and_action( transform_bridge_v1_config_to_action_config( BridgeV1Conf, ConnectorName, ConnectorFields ), - {ActionMap0, OrgActionType} + {ActionMap0, OrgActionType, action} end, {BridgeType, BridgeName, ActionMap, ActionType, ActionOrSource, ConnectorName, ConnectorMap, ConnectorType}. From 7befe898d09fd9546731f7ec6a226c8e4d8300f8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:44:19 -0300 Subject: [PATCH 06/28] fix(mqtt_bridge): fix schema --- .../src/emqx_bridge_mqtt_connector_schema.erl | 13 ++------- .../emqx_bridge_mqtt_pubsub_action_info.erl | 29 +++++++++++-------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index e863d2a2e..ba4373313 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -76,14 +76,6 @@ fields("config_connector") -> } )}, {description, emqx_schema:description_schema()}, - {resource_opts, - mk( - hoconsc:ref(creation_opts), - #{ - required => false, - desc => ?DESC(emqx_resource_schema, "creation_opts") - } - )}, {pool_size, fun egress_pool_size/1} % {ingress, % mk( @@ -95,8 +87,9 @@ fields("config_connector") -> % desc => ?DESC("ingress_desc") % } % )} - ] ++ fields("server_configs"); -fields(creation_opts) -> + ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + fields("server_configs"); +fields(resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields("server_configs") -> [ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 0a1eefd82..de39fc9b4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -52,13 +52,8 @@ make_connector_config_from_bridge_v1_config(Config) -> || {FieldName, _} <- ConnectorConfigSchema ], ConnectorConfigMap = maps:with(ConnectorTopFields, Config), - ResourceOptsSchema = emqx_bridge_mqtt_connector_schema:fields(creation_opts), - ResourceOptsTopFields = [ - erlang:atom_to_binary(FieldName, utf8) - || {FieldName, _} <- ResourceOptsSchema - ], ResourceOptsMap = maps:get(<<"resource_opts">>, ConnectorConfigMap, #{}), - ResourceOptsMap2 = maps:with(ResourceOptsTopFields, ResourceOptsMap), + ResourceOptsMap2 = emqx_connector_schema:project_to_connector_resource_opts(ResourceOptsMap), ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), IngressMap0 = maps:get(<<"ingress">>, Config, #{}), EgressMap = maps:get(<<"egress">>, Config, #{}), @@ -190,7 +185,13 @@ connector_action_config_to_bridge_v1_config( Params = maps:get(<<"parameters">>, ActionConfig, #{}), ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), - ResourceOpts = maps:merge(ResourceOptsConnector, ResourceOptsAction), + ResourceOpts0 = maps:merge(ResourceOptsConnector, ResourceOptsAction), + V1ResourceOptsFields = + lists:map( + fun({Field, _}) -> atom_to_binary(Field) end, + emqx_bridge_mqtt_schema:fields("creation_opts") + ), + ResourceOpts = maps:with(V1ResourceOptsFields, ResourceOpts0), %% Check the direction of the action Direction = case maps:get(<<"remote">>, Params) of @@ -212,11 +213,15 @@ connector_action_config_to_bridge_v1_config( #{<<"egress">> => Parms3}; {<<"publisher">>, LocalT} -> #{ - <<"egress">> => Parms3, - <<"local">> => - #{ - <<"topic">> => LocalT - } + <<"egress">> => + maps:merge( + Parms3, #{ + <<"local">> => + #{ + <<"topic">> => LocalT + } + } + ) }; {<<"subscriber">>, _} -> #{<<"ingress">> => Parms3} From 14b99737e9fbe63c122fb908b9da0a0d15bfc00f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:59:02 -0300 Subject: [PATCH 07/28] fix(mqtt_bridge): add missing fields to POST api spec; fix test --- .../test/emqx_bridge_confluent_tests.erl | 2 +- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl | 2 +- apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl index a7efebf89..2ea7fed22 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl @@ -72,7 +72,7 @@ parse(Hocon) -> Conf. check(SchemaMod, Conf) when is_map(Conf) -> - hocon_tconf:check_plain(SchemaMod, Conf). + hocon_tconf:check_plain(SchemaMod, Conf, #{required => false}). check_action(Conf) when is_map(Conf) -> check(emqx_bridge_v2_schema, Conf). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 2aba6e8ea..67dc52911 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -91,7 +91,7 @@ fields("get_connector") -> fields("get_bridge_v2") -> fields("mqtt_publisher_action"); fields("post_bridge_v2") -> - fields("mqtt_publisher_action"); + fields("mqtt_publisher_action") ++ emqx_bridge_schema:type_and_name_fields(mqtt); fields("put_bridge_v2") -> fields("mqtt_publisher_action"); fields(What) -> diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 0cefd6af4..a74e80fa1 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -166,11 +166,6 @@ common_init(Config0) -> #{work_dir => emqx_cth_suite:work_dir(Config0)} ), {ok, _Api} = emqx_common_test_http:create_default_app(), - - %% ok = emqx_common_test_helpers:start_apps([emqx, emqx_postgresql, emqx_conf, emqx_bridge]), - %% _ = emqx_bridge_enterprise:module_info(), - %% emqx_mgmt_api_test_util:init_suite(), - % Connect to pgsql directly and create the table connect_and_create_table(Config0), {Name, PGConf} = pgsql_config(BridgeType, Config0), From cc34660ab99590f8049eb91b41a65a28759b8ecc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:04:39 -0300 Subject: [PATCH 08/28] fix(actions): use backward-compatible ids --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 66d4dc674..ca421adea 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -939,7 +939,12 @@ id_with_root_name(RootName, BridgeType, BridgeName) -> ) end. -id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName) -> +id_with_root_name(RootName0, BridgeType, BridgeName, ConnectorName) -> + RootName = + case bin(RootName0) of + <<"actions">> -> <<"action">>; + <<"sources">> -> <<"source">> + end, ConnectorType = bin(connector_type(BridgeType)), << (bin(RootName))/binary, From 697c8f5ee11072cc5669ba08aedbf83133289c2e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:34:10 -0300 Subject: [PATCH 09/28] test: fix broken tests --- apps/emqx_bridge/src/emqx_bridge.erl | 14 ++++++++------ apps/emqx_bridge/src/emqx_bridge_v2.erl | 1 + apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 17 ++++------------- .../emqx_bridge/test/emqx_bridge_v2_testlib.erl | 10 ++++++++-- .../test/emqx_bridge_http_SUITE.erl | 12 +++++++++--- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index c7d9a2d27..7df55c81c 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -347,14 +347,16 @@ lookup(Type, Name, RawConf) -> }} end. -get_metrics(Type, Name) -> - case emqx_bridge_v2:is_bridge_v2_type(Type) of +get_metrics(ActionType, Name) -> + case emqx_bridge_v2:is_bridge_v2_type(ActionType) of true -> - case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of + case emqx_bridge_v2:bridge_v1_is_valid(ActionType, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), + BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(ActionType), try - ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one(Type, Name), + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( + ActionType, Name + ), emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) catch error:Reason -> @@ -364,7 +366,7 @@ get_metrics(Type, Name) -> {error, not_bridge_v1_compatible} end; false -> - emqx_resource:get_metrics(emqx_bridge_resource:resource_id(Type, Name)) + emqx_resource:get_metrics(emqx_bridge_resource:resource_id(ActionType, Name)) end. maybe_upgrade(mqtt, Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index ca421adea..38eb56e0f 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -46,6 +46,7 @@ %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies remove/2, + remove/3, %% The following is the remove function that is called by the HTTP API %% It also checks for rule action dependencies and optionally removes %% them diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 82efc77d2..1314fef48 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -188,18 +188,7 @@ end_per_testcase(_, Config) -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_bridge:remove(Type, Name) @@ -1026,9 +1015,11 @@ t_with_redact_update(Config) -> BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), + %% bridge is migrated after creation + ConfigRootKey = connectors, ?assertEqual( Password, - get_raw_config([bridges, Type, Name, password], Config) + get_raw_config([ConfigRootKey, Type, Name, password], Config) ), %% probe with new password; should not be considered redacted diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index eb8a9a5f8..5b821fea4 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -96,9 +96,15 @@ delete_all_bridges_and_connectors() -> delete_all_bridges() -> lists:foreach( fun(#{name := Name, type := Type}) -> - emqx_bridge_v2:remove(Type, Name) + emqx_bridge_v2:remove(actions, Type, Name) end, - emqx_bridge_v2:list() + emqx_bridge_v2:list(actions) + ), + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge_v2:remove(sources, Type, Name) + end, + emqx_bridge_v2:list(sources) ). delete_all_connectors() -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 3b7303300..f21e879b8 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -244,6 +244,12 @@ parse_http_request_assertive(ReqStr0) -> %% Helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +get_metrics(Name) -> + %% Note: `emqx_bridge:get_metrics/2' is currently *only* called in prod by + %% `emqx_bridge_api:lookup_from_local_node' with an action (not v1 bridge) type. + Type = <<"http">>, + emqx_bridge:get_metrics(Type, Name). + bridge_async_config(#{port := Port} = Config) -> Type = maps:get(type, Config, ?BRIDGE_TYPE), Name = maps:get(name, Config, ?BRIDGE_NAME), @@ -570,7 +576,7 @@ t_path_not_found(Config) -> success := 0 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -611,7 +617,7 @@ t_too_many_requests(Config) -> success := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -654,7 +660,7 @@ t_rule_action_expired(Config) -> dropped := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ?retry( From ab1b0dda677d09e169e2bf9a7ca7b623a573009a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:35:42 -0300 Subject: [PATCH 10/28] refactor: fix typo --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 38eb56e0f..c46013c0c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1345,13 +1345,13 @@ get_conf_root_key_if_only_one(BridgeType, BridgeName) -> LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), case {LookUpConfActions, LookUpConfSources} of {{error, bridge_not_found}, {error, bridge_not_found}} -> - error({action_or_soruces_not_found, BridgeType, BridgeName}); + error({action_or_source_not_found, BridgeType, BridgeName}); {{error, bridge_not_found}, _Conf} -> ?ROOT_KEY_SOURCES; {_Conf, {error, bridge_not_found}} -> ?ROOT_KEY_ACTIONS; {_Conf1, _Conf2} -> - error({name_clash_action_soruces, BridgeType, BridgeName}) + error({name_clash_action_source, BridgeType, BridgeName}) end. lookup_conf(RootName, Type, Name) -> From 3597ee7c939dbe1db421cfb8f66f25a00425ab37 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 19:00:47 -0300 Subject: [PATCH 11/28] fix(mqtt_action): fix resource_opts schema --- .../src/emqx_bridge_mqtt_pubsub_action_info.erl | 4 ++-- .../emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index de39fc9b4..8918a60be 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -88,7 +88,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_publisher_action"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), @@ -113,7 +113,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 67dc52911..2cc13daaf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -80,11 +80,11 @@ fields(ingress_parameters) -> Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), Fields1 = proplists:delete(pool_size, Fields0), Fields1; -fields("resource_opts") -> +fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_resource_schema:fields("creation_opts") + emqx_bridge_v2_schema:resource_opts_fields() ); fields("get_connector") -> emqx_bridge_mqtt_connector_schema:fields("config_connector"); @@ -105,7 +105,7 @@ fields(What) -> desc("config") -> ?DESC("desc_config"); -desc("resource_opts") -> +desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; From 1ad3100cad9a6398aa549212944c308838aab3e9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 20:02:04 -0300 Subject: [PATCH 12/28] chore: add i18n --- .../src/emqx_bridge_mqtt_connector_schema.erl | 4 ++++ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 8 +++++++ .../emqx_bridge_mqtt_connector_schema.hocon | 5 +++++ rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon | 22 +++++++++++++++++++ rel/i18n/emqx_bridge_v2_schema.hocon | 10 +++++++-- 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index ba4373313..83be577f4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -338,6 +338,8 @@ egress_pool_size(Prop) -> desc("server_configs") -> ?DESC("server_configs"); +desc("config_connector") -> + ?DESC("config_connector"); desc("ingress") -> ?DESC("ingress_desc"); desc("ingress_remote") -> @@ -350,6 +352,8 @@ desc("egress_remote") -> ?DESC("egress_remote"); desc("egress_local") -> ?DESC("egress_local"); +desc(resource_opts) -> + ?DESC(emqx_resource_schema, <<"resource_opts">>); desc(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 2cc13daaf..6d075334a 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -107,6 +107,10 @@ desc("config") -> ?DESC("desc_config"); desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); +desc(action_parameters) -> + ?DESC(action_parameters); +desc(ingress_parameters) -> + ?DESC(ingress_parameters); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; desc("config_connector") -> @@ -115,6 +119,10 @@ desc("http_action") -> ?DESC("desc_config"); desc("parameters_opts") -> ?DESC("config_parameters_opts"); +desc("mqtt_publisher_action") -> + ?DESC("mqtt_publisher_action"); +desc("mqtt_subscriber_source") -> + ?DESC("mqtt_subscriber_source"); desc(_) -> undefined. diff --git a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon index 7c7bf68c9..25bb8aad2 100644 --- a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon @@ -194,4 +194,9 @@ username.desc: username.label: """Username""" +config_connector.desc: +"""Configurations for an MQTT connector.""" +config_connector.label: +"""MQTT connector""" + } diff --git a/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon new file mode 100644 index 000000000..637189781 --- /dev/null +++ b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon @@ -0,0 +1,22 @@ +emqx_bridge_mqtt_pubsub_schema { + action_parameters.desc: + """Action specific configs.""" + action_parameters.label: + """Action""" + + ingress_parameters.desc: + """Source specific configs.""" + ingress_parameters.label: + """Source""" + + mqtt_publisher_action.desc: + """Action configs.""" + mqtt_publisher_action.label: + """Action""" + + mqtt_subscriber_source.desc: + """Source configs.""" + mqtt_subscriber_source.label: + """Source""" + +} diff --git a/rel/i18n/emqx_bridge_v2_schema.hocon b/rel/i18n/emqx_bridge_v2_schema.hocon index 69f8a9109..3a4bf6140 100644 --- a/rel/i18n/emqx_bridge_v2_schema.hocon +++ b/rel/i18n/emqx_bridge_v2_schema.hocon @@ -1,10 +1,16 @@ emqx_bridge_v2_schema { desc_bridges_v2.desc: -"""Configuration for bridges.""" +"""Configuration for actions.""" desc_bridges_v2.label: -"""Bridge Configuration""" +"""Action Configuration""" + +desc_sources.desc: +"""Configuration for sources.""" + +desc_sources.label: +"""Source Configuration""" mqtt_topic.desc: """MQTT topic or topic filter as data source (action input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in the remote system.""" From 7fc069da46302afb8c240ef10c8cf0810777c63a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 20:02:54 -0300 Subject: [PATCH 13/28] test: fix another broken test --- apps/emqx_bridge/test/emqx_bridge_SUITE.erl | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index 30107d0ce..19ae516ec 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -32,7 +32,9 @@ init_per_suite(Config) -> emqx_conf, emqx_connector, emqx_bridge_http, - emqx_bridge + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine ], #{work_dir => ?config(priv_dir, Config)} ), @@ -154,14 +156,18 @@ setup_fake_telemetry_data() -> ok. t_update_ssl_conf(Config) -> - Path = proplists:get_value(config_path, Config), - CertDir = filename:join([emqx:mutable_certs_dir() | Path]), + [_Root, Type, Name] = proplists:get_value(config_path, Config), + CertDir = filename:join([emqx:mutable_certs_dir(), connectors, Type, Name]), EnableSSLConf = #{ <<"bridge_mode">> => false, <<"clean_start">> => true, <<"keepalive">> => <<"60s">>, <<"proto_ver">> => <<"v4">>, <<"server">> => <<"127.0.0.1:1883">>, + <<"egress">> => #{ + <<"local">> => #{<<"topic">> => <<"t">>}, + <<"remote">> => #{<<"topic">> => <<"remote/t">>} + }, <<"ssl">> => #{ <<"cacertfile">> => cert_file("cafile"), @@ -171,10 +177,15 @@ t_update_ssl_conf(Config) -> <<"verify">> => <<"verify_peer">> } }, - {ok, _} = emqx:update_config(Path, EnableSSLConf), + CreateCfg = [ + {bridge_name, Name}, + {bridge_type, Type}, + {bridge_config, #{}} + ], + {ok, _} = emqx_bridge_testlib:create_bridge_api(CreateCfg, EnableSSLConf), ?assertMatch({ok, [_, _, _]}, file:list_dir(CertDir)), NoSSLConf = EnableSSLConf#{<<"ssl">> := #{<<"enable">> => false}}, - {ok, _} = emqx:update_config(Path, NoSSLConf), + {ok, _} = emqx_bridge_testlib:update_bridge_api(CreateCfg, NoSSLConf), {ok, _} = emqx_tls_certfile_gc:force(), ?assertMatch({error, enoent}, file:list_dir(CertDir)), ok. From 862283ff7c7d5d12ae236ce256729e440dce0b49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 20 Dec 2023 13:10:50 -0300 Subject: [PATCH 14/28] test: fix expected connector name after name convention generation changed --- .../test/emqx_bridge_gcp_pubsub_producer_SUITE.erl | 8 ++++++-- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl index e030bbc74..92e4e09a6 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl @@ -159,7 +159,7 @@ generate_config(Config0) -> } = gcp_pubsub_config(Config0), %% FIXME %% `emqx_bridge_resource:resource_id' requires an existing connector in the config..... - ConnectorName = <<"connector_", ActionName/binary>>, + ConnectorName = ActionName, ConnectorResourceId = <<"connector:", ?CONNECTOR_TYPE_BIN/binary, ":", ConnectorName/binary>>, ActionResourceId = emqx_bridge_v2:id(?ACTION_TYPE_BIN, ActionName, ConnectorName), BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_V1_TYPE_BIN, ActionName), @@ -1228,7 +1228,11 @@ do_econnrefused_or_timeout_test(Config, Error) -> %% _Msg = "The connection was lost." ok; Trace0 -> - error({unexpected_trace, Trace0}) + error( + {unexpected_trace, Trace0, #{ + expected_connector_id => ConnectorResourceId + }} + ) end; timeout -> ?assertMatch( diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index bb3ca87b7..6e520ba58 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -412,7 +412,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"webhook_name">> => #{ - <<"connector">> => <<"connector_webhook_name">>, + <<"connector">> => <<"webhook_name">>, <<"description">> => <<>>, <<"enable">> => true, <<"parameters">> => @@ -440,7 +440,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"http">> => #{ - <<"connector_webhook_name">> => + <<"webhook_name">> => #{ <<"connect_timeout">> => <<"15s">>, <<"description">> => <<>>, From 12dc9fbeb9532f6e4fd0dd35ee5f27e8e02862ed Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 20 Dec 2023 15:45:51 -0300 Subject: [PATCH 15/28] test(mqtt_bridge): fix assertion --- .../test/emqx_bridge_mqtt_SUITE.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index bd3fb68de..3c50e16d8 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -410,13 +410,16 @@ t_mqtt_egress_bridge_warns_clean_start(_) -> ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []) + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + + ok end, - ?wait_async_action( - Action(), - #{?snk_kind := mqtt_clean_start_egress_action_warning}, - 10000 - ), + {ok, {ok, _}} = + ?wait_async_action( + Action(), + #{?snk_kind := mqtt_clean_start_egress_action_warning}, + 10000 + ), ok. t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> From 6511693b2ee139216a93cf81f8a719602d691f40 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 21 Dec 2023 14:14:41 -0300 Subject: [PATCH 16/28] refactor(action_api): prepare for `/sources` HTTP API --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_bridge/src/emqx_bridge_api.erl | 24 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 42 +++- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 216 ++++++++++++------ .../src/proto/emqx_bridge_proto_v6.erl | 196 ++++++++++++++++ .../test/emqx_bridge_api_SUITE.erl | 7 +- apps/emqx_bridge/test/emqx_bridge_testlib.erl | 16 +- .../test/emqx_bridge_v2_testlib.erl | 18 +- 8 files changed, 404 insertions(+), 116 deletions(-) create mode 100644 apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 9721a7f2f..9bd824242 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -8,6 +8,7 @@ {emqx_bridge,3}. {emqx_bridge,4}. {emqx_bridge,5}. +{emqx_bridge,6}. {emqx_broker,1}. {emqx_cm,1}. {emqx_cm,2}. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 3168ae590..8f36fd700 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,6 +49,11 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). +%% only for testting/mocking +-export([supported_versions/1]). + +-define(BPAPI_NAME, emqx_bridge). + -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). @@ -1102,18 +1107,18 @@ maybe_try_restart(_, _, _) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> case lists:member(Node, mria:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. do_bpapi_call_vsn(SupportedVersion, Call, Args) -> - case lists:member(SupportedVersion, supported_versions(Call)) of + case lists:member(SupportedVersion, ?MODULE:supported_versions(Call)) of true -> apply(emqx_bridge_proto_v4, Call, Args); false -> @@ -1125,10 +1130,15 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2, 3, 4, 5]; -supported_versions(start_bridges_to_all_nodes) -> [2, 3, 4, 5]; -supported_versions(get_metrics_from_all_nodes) -> [4, 5]; -supported_versions(_Call) -> [1, 2, 3, 4, 5]. +supported_versions(start_bridge_to_node) -> bpapi_version_range(2, latest); +supported_versions(start_bridges_to_all_nodes) -> bpapi_version_range(2, latest); +supported_versions(get_metrics_from_all_nodes) -> bpapi_version_range(4, latest); +supported_versions(_Call) -> bpapi_version_range(1, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). redact(Term) -> emqx_utils:redact(Term). diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index c46013c0c..fb8ae2e2e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -43,6 +43,7 @@ lookup/2, lookup/3, create/3, + create/4, %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies remove/2, @@ -50,7 +51,8 @@ %% The following is the remove function that is called by the HTTP API %% It also checks for rule action dependencies and optionally removes %% them - check_deps_and_remove/3 + check_deps_and_remove/3, + check_deps_and_remove/4 ]). %% Operations @@ -62,9 +64,11 @@ send_message/4, query/4, start/2, + start/3, reset_metrics/2, reset_metrics/3, create_dry_run/2, + create_dry_run/3, get_metrics/2, get_metrics/3 ]). @@ -150,6 +154,10 @@ -type bridge_v2_type() :: binary() | atom() | [byte()]. -type bridge_v2_name() :: binary() | atom() | [byte()]. +-type root_cfg_key() :: ?ROOT_KEY_ACTIONS | ?ROOT_KEY_SOURCES. + +-export_type([root_cfg_key/0]). + %%==================================================================== %%==================================================================== @@ -212,7 +220,7 @@ unload_bridges(ConfRooKey) -> lookup(Type, Name) -> lookup(?ROOT_KEY_ACTIONS, Type, Name). --spec lookup(sources | actions, bridge_v2_type(), bridge_v2_name()) -> +-spec lookup(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(ConfRootName, Type, Name) -> case emqx:get_raw_config([ConfRootName, Type, Name], not_found) of @@ -315,6 +323,11 @@ remove(ConfRootKey, BridgeType, BridgeName) -> -spec check_deps_and_remove(bridge_v2_type(), bridge_v2_name(), boolean()) -> ok | {error, any()}. check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> + check_deps_and_remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, AlsoDeleteActions). + +-spec check_deps_and_remove(root_cfg_key(), bridge_v2_type(), bridge_v2_name(), boolean()) -> + ok | {error, any()}. +check_deps_and_remove(ConfRooKey, BridgeType, BridgeName, AlsoDeleteActions) -> AlsoDelete = case AlsoDeleteActions of true -> [rule_actions]; @@ -328,7 +341,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> ) of ok -> - remove(BridgeType, BridgeName); + remove(ConfRooKey, BridgeType, BridgeName); {error, Reason} -> {error, Reason} end. @@ -539,11 +552,14 @@ disable_enable(ConfRootKey, Action, BridgeType, BridgeName) when ?ENABLE_OR_DISA %% is something else than connected after starting the connector or if an %% error occurred when the connector was started. -spec start(term(), term()) -> ok | {error, Reason :: term()}. -start(BridgeV2Type, Name) -> +start(ActionOrSourceType, Name) -> + start(?ROOT_KEY_ACTIONS, ActionOrSourceType, Name). + +-spec start(root_cfg_key(), term(), term()) -> ok | {error, Reason :: term()}. +start(ConfRootKey, BridgeV2Type, Name) -> ConnectorOpFun = fun(ConnectorType, ConnectorName) -> emqx_connector_resource:start(ConnectorType, ConnectorName) end, - ConfRootKey = ?ROOT_KEY_ACTIONS, connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, true). connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> @@ -694,10 +710,15 @@ health_check(ConfRootKey, BridgeType, BridgeName) -> end. -spec create_dry_run(bridge_v2_type(), Config :: map()) -> ok | {error, term()}. -create_dry_run(Type, Conf0) -> +create_dry_run(Type, Conf) -> + create_dry_run(?ROOT_KEY_ACTIONS, Type, Conf). + +-spec create_dry_run(root_cfg_key(), bridge_v2_type(), Config :: map()) -> ok | {error, term()}. +create_dry_run(ConfRootKey, Type, Conf0) -> Conf1 = maps:without([<<"name">>], Conf0), TypeBin = bin(Type), - RawConf = #{<<"actions">> => #{TypeBin => #{<<"temp_name">> => Conf1}}}, + ConfRootKeyBin = bin(ConfRootKey), + RawConf = #{ConfRootKeyBin => #{TypeBin => #{<<"temp_name">> => Conf1}}}, %% Check config try _ = @@ -722,6 +743,9 @@ create_dry_run(Type, Conf0) -> end. create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> + create_dry_run_helper(?ROOT_KEY_ACTIONS, BridgeType, ConnectorRawConf, BridgeV2RawConf). + +create_dry_run_helper(ConfRootKey, BridgeType, ConnectorRawConf, BridgeV2RawConf) -> BridgeName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), ConnectorType = connector_type(BridgeType), OnReadyCallback = @@ -730,7 +754,7 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> ChannelTestId = id(BridgeType, BridgeName, ConnectorName), Conf = emqx_utils_maps:unsafe_atom_key_map(BridgeV2RawConf), AugmentedConf = augment_channel_config( - ?ROOT_KEY_ACTIONS, + ConfRootKey, BridgeType, BridgeName, Conf @@ -756,6 +780,8 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> get_metrics(Type, Name) -> get_metrics(?ROOT_KEY_ACTIONS, Type, Name). +-spec get_metrics(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> + emqx_metrics_worker:metrics(). get_metrics(ConfRootKey, Type, Name) -> emqx_resource:get_metrics(id_with_root_name(ConfRootKey, Type, Name)). diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 254390a36..3f0d18fae 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -26,6 +26,9 @@ -import(hoconsc, [mk/2, array/1, enum/1]). -import(emqx_utils, [redact/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% Swagger specs from hocon schema -export([ api_spec/0, @@ -48,7 +51,14 @@ ]). %% BpAPI / RPC Targets --export([lookup_from_local_node/2, get_metrics_from_local_node/2]). +-export([ + lookup_from_local_node/2, + get_metrics_from_local_node/2, + lookup_from_local_node_v6/3, + get_metrics_from_local_node_v6/3 +]). + +-define(BPAPI_NAME, emqx_bridge). -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( @@ -393,16 +403,51 @@ schema("/action_types") -> }. '/actions'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, _} -> - ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); - {error, not_found} -> - Conf = filter_out_request_body(Conf0), - create_bridge(BridgeType, BridgeName, Conf) - end; + handle_create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, Conf0); '/actions'(get, _Params) -> - Nodes = mria:running_nodes(), - NodeReplies = emqx_bridge_proto_v5:v2_list_bridges_on_nodes(Nodes), + handle_list(?ROOT_KEY_ACTIONS). + +'/actions/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, 200)); +'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_ACTIONS, Id, Conf0); +'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_ACTIONS, Id, Qs). + +'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName)). + +'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_ACTIONS, Id). + +'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_ACTIONS, Id, Enable). + +'/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_ACTIONS, Id, Op). + +'/nodes/:node/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_ACTIONS, Node, Id, Op). + +'/actions_probe'(post, Request) -> + handle_probe(?ROOT_KEY_ACTIONS, Request). + +'/action_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:types()). + +%%------------------------------------------------------------------------------ +%% Handlers +%%------------------------------------------------------------------------------ + +handle_list(ConfRootKey) -> + Nodes = emqx:running_nodes(), + NodeReplies = emqx_bridge_proto_v6:v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey), case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ @@ -414,34 +459,44 @@ schema("/action_types") -> ?INTERNAL_ERROR(Reason) end. -'/actions/:id'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); -'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> +handle_create(ConfRootKey, Type, Name, Conf0) -> + case emqx_bridge_v2:lookup(ConfRootKey, Type, Name) of + {ok, _} -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); + {error, not_found} -> + Conf = filter_out_request_body(Conf0), + create_bridge(ConfRootKey, Type, Name, Conf) + end. + +handle_update(ConfRootKey, Id, Conf0) -> Conf1 = filter_out_request_body(Conf0), ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), Conf = emqx_utils:deobfuscate(Conf1, RawConf), - update_bridge(BridgeType, BridgeName, Conf); + update_bridge(ConfRootKey, BridgeType, BridgeName, Conf); {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end - ); -'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + ). + +handle_delete(ConfRootKey, Id, QueryStringOpts) -> ?TRY_PARSE_ID( Id, case emqx_bridge_v2:lookup(BridgeType, BridgeName) of {ok, _} -> AlsoDeleteActions = - case maps:get(<<"also_delete_dep_actions">>, Qs, <<"false">>) of + case maps:get(<<"also_delete_dep_actions">>, QueryStringOpts, <<"false">>) of <<"true">> -> true; true -> true; _ -> false end, case - emqx_bridge_v2:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) + emqx_bridge_v2:check_deps_and_remove( + ConfRootKey, BridgeType, BridgeName, AlsoDeleteActions + ) of ok -> ?NO_CONTENT; @@ -465,23 +520,22 @@ schema("/action_types") -> end ). -'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). - -'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> +handle_reset_metrics(ConfRootKey, Id) -> ?TRY_PARSE_ID( Id, begin ActionType = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(ActionType, BridgeName), + ok = emqx_bridge_v2:reset_metrics(ConfRootKey, ActionType, BridgeName), ?NO_CONTENT end ). -'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> +handle_disable_enable(ConfRootKey, Id, Enable) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:disable_enable(enable_func(Enable), BridgeType, BridgeName) of + case + emqx_bridge_v2:disable_enable(ConfRootKey, enable_func(Enable), BridgeType, BridgeName) + of {ok, _} -> ?NO_CONTENT; {error, {pre_config_update, _, bridge_not_found}} -> @@ -495,41 +549,37 @@ schema("/action_types") -> end ). -'/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op} -}) -> +handle_operation(ConfRootKey, Id, Op) -> ?TRY_PARSE_ID( Id, begin OperFunc = operation_func(all, Op), - Nodes = mria:running_nodes(), - call_operation_if_enabled(all, OperFunc, [Nodes, BridgeType, BridgeName]) + Nodes = emqx:running_nodes(), + call_operation_if_enabled(all, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) end ). -'/nodes/:node/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op, node := Node} -}) -> +handle_node_operation(ConfRootKey, Node, Id, Op) -> ?TRY_PARSE_ID( Id, case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> OperFunc = operation_func(TargetNode, Op), - call_operation_if_enabled(TargetNode, OperFunc, [TargetNode, BridgeType, BridgeName]); + call_operation_if_enabled(TargetNode, OperFunc, [ + TargetNode, ConfRootKey, BridgeType, BridgeName + ]); {error, _} -> ?NOT_FOUND(<<"Invalid node name: ", Node/binary>>) end ). -'/actions_probe'(post, Request) -> +handle_probe(ConfRootKey, Request) -> RequestMeta = #{module => ?MODULE, method => post, path => "/actions_probe"}, case emqx_dashboard_swagger:filter_check_request_and_translate_body(Request, RequestMeta) of - {ok, #{body := #{<<"type">> := ConnType} = Params}} -> + {ok, #{body := #{<<"type">> := Type} = Params}} -> Params1 = maybe_deobfuscate_bridge_probe(Params), Params2 = maps:remove(<<"type">>, Params1), - case emqx_bridge_v2:create_dry_run(ConnType, Params2) of + case emqx_bridge_v2:create_dry_run(ConfRootKey, Type, Params2) of ok -> ?NO_CONTENT; {error, #{kind := validation_error} = Reason0} -> @@ -548,9 +598,7 @@ schema("/action_types") -> redact(BadRequest) end. -'/action_types'(get, _Request) -> - ?OK(emqx_bridge_v2_schema:types()). - +%%% API helpers maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) -> case emqx_bridge_v2:lookup(ActionType, BridgeName) of {ok, #{raw_config := RawConf}} -> @@ -564,7 +612,6 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeN maybe_deobfuscate_bridge_probe(Params) -> Params. -%%% API helpers is_ok(ok) -> ok; is_ok(OkResult = {ok, _}) -> @@ -587,9 +634,16 @@ is_ok(ResL) -> end. %% bridge helpers -lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> - Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v5:v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of +-spec lookup_from_all_nodes(emqx_bridge_v2:root_cfg_key(), _, _, _) -> _. +lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, SuccCode) -> + Nodes = emqx:running_nodes(), + case + is_ok( + emqx_bridge_proto_v6:v2_lookup_from_all_nodes_v6( + Nodes, ConfRootKey, BridgeType, BridgeName + ) + ) + of {ok, [{ok, _} | _] = Results} -> {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> @@ -598,10 +652,10 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> ?INTERNAL_ERROR(Reason) end. -get_metrics_from_all_nodes(ActionType, ActionName) -> +get_metrics_from_all_nodes(ConfRootKey, Type, Name) -> Nodes = emqx:running_nodes(), Result = maybe_unwrap( - emqx_bridge_proto_v5:v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) + emqx_bridge_proto_v6:v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, Type, Name) ), case Result of Metrics when is_list(Metrics) -> @@ -610,22 +664,25 @@ get_metrics_from_all_nodes(ActionType, ActionName) -> ?INTERNAL_ERROR(Reason) end. -operation_func(all, start) -> v2_start_bridge_to_all_nodes; -operation_func(_Node, start) -> v2_start_bridge_to_node. +operation_func(all, start) -> v2_start_bridge_to_all_nodes_v6; +operation_func(_Node, start) -> v2_start_bridge_to_node_v6; +operation_func(all, lookup) -> v2_lookup_from_all_nodes_v6; +operation_func(all, list) -> v2_list_bridges_on_nodes_v6; +operation_func(all, get_metrics) -> v2_get_metrics_from_all_nodes_v6. -call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) -> - try is_enabled_bridge(BridgeType, BridgeName) of +call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) -> + try is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) of false -> ?BRIDGE_NOT_ENABLED; true -> - call_operation(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) + call_operation(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) catch throw:not_found -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end. -is_enabled_bridge(BridgeType, BridgeName) -> - try emqx_bridge_v2:lookup(BridgeType, binary_to_existing_atom(BridgeName)) of +is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) -> + try emqx_bridge_v2:lookup(ConfRootKey, BridgeType, binary_to_existing_atom(BridgeName)) of {ok, #{raw_config := ConfMap}} -> maps:get(<<"enable">>, ConfMap, false); {error, not_found} -> @@ -637,7 +694,7 @@ is_enabled_bridge(BridgeType, BridgeName) -> throw(not_found) end. -call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> +call_operation(NodeOrAll, OperFunc, Args = [_Nodes, _ConfRootKey, BridgeType, BridgeName]) -> case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok -> ?NO_CONTENT; @@ -668,12 +725,12 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> - case lists:member(Node, mria:running_nodes()) of + case lists:member(Node, emqx:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. @@ -681,7 +738,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(Version, Call, Args) -> case is_supported_version(Version, Call) of true -> - apply(emqx_bridge_proto_v5, Call, Args); + apply(emqx_bridge_proto_v6, Call, Args); false -> {error, not_implemented} end. @@ -689,7 +746,12 @@ do_bpapi_call_vsn(Version, Call, Args) -> is_supported_version(Version, Call) -> lists:member(Version, supported_versions(Call)). -supported_versions(_Call) -> [5]. +supported_versions(_Call) -> bpapi_version_range(6, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). maybe_unwrap({error, not_implemented}) -> {error, not_implemented}; @@ -767,10 +829,22 @@ lookup_from_local_node(BridgeType, BridgeName) -> Error -> Error end. +%% RPC Target +-spec lookup_from_local_node_v6(emqx_bridge_v2:root_cfg_key(), _, _) -> _. +lookup_from_local_node_v6(ConfRootKey, BridgeType, BridgeName) -> + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of + {ok, Res} -> {ok, format_resource(Res, node())}; + Error -> Error + end. + %% RPC Target get_metrics_from_local_node(ActionType, ActionName) -> format_metrics(emqx_bridge_v2:get_metrics(ActionType, ActionName)). +%% RPC Target +get_metrics_from_local_node_v6(ConfRootKey, Type, Name) -> + format_metrics(emqx_bridge_v2:get_metrics(ConfRootKey, Type, Name)). + %% resource format_resource( #{ @@ -938,13 +1012,13 @@ format_resource_data(error, Error, Result) -> format_resource_data(K, V, Result) -> Result#{K => V}. -create_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 201). +create_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 201). -update_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 200). +update_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 200). -create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> +create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> Check = try is_binary(BridgeType) andalso emqx_resource:validate_type(BridgeType), @@ -955,15 +1029,15 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> end, case Check of ok -> - do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode); + do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode); BadRequest -> BadRequest end. -do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> - case emqx_bridge_v2:create(BridgeType, BridgeName, Conf) of +do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> + case emqx_bridge_v2:create(ConfRootKey, BridgeType, BridgeName, Conf) of {ok, _} -> - lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode); + lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, HttpStatusCode); {error, {PreOrPostConfigUpdate, _HandlerMod, Reason}} when PreOrPostConfigUpdate =:= pre_config_update; PreOrPostConfigUpdate =:= post_config_update diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl new file mode 100644 index 000000000..d6fe68466 --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v6). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + get_metrics_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3, + + %% introduced in v6 + v2_lookup_from_all_nodes_v6/4, + v2_list_bridges_on_nodes_v6/2, + v2_get_metrics_from_all_nodes_v6/4, + v2_start_bridge_to_node_v6/4, + v2_start_bridge_to_all_nodes_v6/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.5.0". + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(term()). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()). +get_metrics_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + get_metrics_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +%%-------------------------------------------------------------------------------- +%% introduced in v6 +%%-------------------------------------------------------------------------------- + +%% V2 Calls +-spec v2_lookup_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_lookup_from_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + lookup_from_local_node_v6, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_list_bridges_on_nodes_v6([node()], emqx_bridge_v2:root_cfg_key()) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey) -> + erpc:multicall(Nodes, emqx_bridge_v2, list, [ConfRootKey], ?TIMEOUT). + +-spec v2_get_metrics_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, ActionType, ActionName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + get_metrics_from_local_node_v6, + [ConfRootKey, ActionType, ActionName], + ?TIMEOUT + ). + +-spec v2_start_bridge_to_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(ok). +v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_start_bridge_to_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> + term(). +v2_start_bridge_to_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 1314fef48..112e24e63 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -160,8 +160,9 @@ end_per_group(_, Config) -> init_per_testcase(t_broken_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), - meck:expect(emqx_bpapi, supported_version, 1, -1), meck:expect(emqx_bpapi, supported_version, 2, -1), + meck:new(emqx_bridge_api, [passthrough]), + meck:expect(emqx_bridge_api, supported_versions, 1, []), init_per_testcase(common, Config); init_per_testcase(t_old_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), @@ -173,10 +174,10 @@ init_per_testcase(_, Config) -> [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config]. end_per_testcase(t_broken_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(t_old_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(_, Config) -> Sock = ?config(sock, Config), diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index df4560e6e..7ab8c68a6 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -196,20 +196,10 @@ delete_bridge_http_api_v1(Opts) -> op_bridge_api(Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, + Method = post, + Params = [], + Res = emqx_bridge_v2_testlib:request(Method, Path, Params), ct:pal("bridge op result: ~p", [Res]), Res. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 5b821fea4..f7dd74161 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -308,21 +308,11 @@ update_bridge_api(Config, Overrides) -> op_bridge_api(Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge op result: ~p", [Res]), + Method = post, + Params = [], + Res = request(Method, Path, Params), + ct:pal("bridge op result:\n ~p", [Res]), Res. probe_bridge_api(Config) -> From e6ccfa5b397e275e6512422d273e07c18637b097 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Jan 2024 12:17:26 -0300 Subject: [PATCH 17/28] fix(mqtt_bridge): fixes after rebasing onto current `master` Rebased on top of 7f57ec47d5f634da67a63ede531b5e85f2a229b6 --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 7 +--- .../src/emqx_bridge_mqtt_connector_schema.erl | 40 ++++++------------- .../emqx_bridge_mqtt_pubsub_action_info.erl | 2 +- .../test/emqx_bridge_mqtt_SUITE.erl | 15 +++---- 5 files changed, 24 insertions(+), 42 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e6fe78ab8..0c00a0d59 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 6f9465cb0..dbdf68ef1 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -414,10 +414,8 @@ mk_client_opts( ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }). -parse_id_to_name(<>) -> - Name; parse_id_to_name(Id) -> - {_Type, Name} = emqx_bridge_resource:parse_bridge_id(Id, #{atom_name => false}), + {_Type, Name} = emqx_connector_resource:parse_connector_id(Id, #{atom_name => false}), Name. mk_client_opt_password(Options = #{password := Secret}) -> @@ -447,7 +445,6 @@ connect(Options) -> }), Name = proplists:get_value(name, Options), WorkerId = proplists:get_value(ecpool_worker_id, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), ClientOpts = proplists:get_value(client_opts, Options), case emqtt:start_link(mk_client_opts(Name, WorkerId, ClientOpts)) of {ok, Pid} -> @@ -475,7 +472,7 @@ mk_client_opts( }. mk_clientid(WorkerId, ClientId) -> - iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + emqx_bridge_mqtt_lib:bytes23([ClientId], WorkerId). mk_client_event_handler(Name, TopicToHandlerIndex) -> #{ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 83be577f4..edbb40cbb 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -38,6 +38,7 @@ -import(hoconsc, [mk/2, ref/2]). +-define(CONNECTOR_TYPE, mqtt). -define(MQTT_HOST_OPTS, #{default_port => 1883}). namespace() -> "connector_mqtt". @@ -66,28 +67,10 @@ fields("config") -> )} ]; fields("config_connector") -> - [ - {enable, - mk( - boolean(), - #{ - desc => <<"Enable or disable this connector">>, - default => true - } - )}, - {description, emqx_schema:description_schema()}, - {pool_size, fun egress_pool_size/1} - % {ingress, - % mk( - % hoconsc:array( - % hoconsc:ref(connector_ingress) - % ), - % #{ - % required => {false, recursively}, - % desc => ?DESC("ingress_desc") - % } - % )} - ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + emqx_connector_schema:common_fields() ++ fields("specific_connector_config"); +fields("specific_connector_config") -> + [{pool_size, fun egress_pool_size/1}] ++ + emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ fields("server_configs"); fields(resource_opts) -> emqx_connector_schema:resource_opts_fields(); @@ -317,12 +300,13 @@ fields("egress_remote") -> } )} ]; -fields("get_connector") -> - fields("config_connector"); -fields("post_connector") -> - fields("config_connector"); -fields("put_connector") -> - fields("config_connector"); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + Fields = fields("specific_connector_config"), + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, Fields); fields(What) -> error({emqx_bridge_mqtt_connector_schema, missing_field_handler, What}). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 8918a60be..e4a4fcd19 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -94,7 +94,7 @@ bridge_v1_config_to_action_config_helper( ), LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), - EgressMap1 = maps:remove(<<"local">>, EgressMap0), + EgressMap1 = maps:without([<<"local">>, <<"pool_size">>], EgressMap0), %% Add parameters field (Egress map) to the action config ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), ConfigMap3 = diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 3c50e16d8..807fba3c9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -564,17 +564,17 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> t_egress_short_clientid(_Config) -> %% Name is short, expect the actual client ID in use is hashed from - %% E: - Name = "abc01234", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc01234">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = iolist_to_binary([BaseId, $:, "1"]), test_egress_clientid(Name, ExpectedClientId). t_egress_long_clientid(_Config) -> %% Expect the actual client ID in use is hashed from - %% E: - Name = "abc01234567890123456789", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc012345678901234567890">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = emqx_bridge_mqtt_lib:bytes23(BaseId, 1), test_egress_clientid(Name, ExpectedClientId). @@ -1049,7 +1049,8 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - emqx_utils_json:decode(Bridge) + emqx_utils_json:decode(Bridge), + #{expected_type => Type, expected_name => Name} ), emqx_bridge_resource:bridge_id(Type, Name). From 28de7c89c7c02bc6114fb62133e5ee11d2b8110d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Jan 2024 17:24:55 -0300 Subject: [PATCH 18/28] feat: add `/sources*` HTTP APIs --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 20 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 23 +- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 384 ++++++++++++++++-- .../src/proto/emqx_bridge_proto_v6.erl | 12 +- .../src/schema/emqx_bridge_v2_schema.erl | 239 ++++++++--- ...qx_bridge_v1_compatibility_layer_SUITE.erl | 2 +- .../test/emqx_bridge_v2_testlib.erl | 187 +++++---- .../emqx_bridge/test/emqx_bridge_v2_tests.erl | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 4 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 23 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 175 ++++++++ 11 files changed, 870 insertions(+), 201 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 0a870abb8..ec7a7431b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -23,6 +23,7 @@ bridge_to_resource_type/1, resource_id/1, resource_id/2, + resource_id/3, bridge_id/2, parse_bridge_id/1, parse_bridge_id/2, @@ -62,6 +63,9 @@ ?IS_BI_DIR_BRIDGE(TYPE) ). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(BridgeType) when is_binary(BridgeType) -> bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8)); @@ -85,11 +89,21 @@ bridge_impl_module(_BridgeType) -> undefined. -endif. resource_id(BridgeId) when is_binary(BridgeId) -> + resource_id_for_kind(?ROOT_KEY_ACTIONS, BridgeId). + +resource_id(BridgeType, BridgeName) -> + resource_id(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +resource_id(ConfRootKey, BridgeType, BridgeName) -> + BridgeId = bridge_id(BridgeType, BridgeName), + resource_id_for_kind(ConfRootKey, BridgeId). + +resource_id_for_kind(ConfRootKey, BridgeId) when is_binary(BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, _Name] -> case emqx_bridge_v2:is_bridge_v2_type(Type) of true -> - emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(BridgeId); + emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId); false -> <<"bridge:", BridgeId/binary>> end; @@ -97,10 +111,6 @@ resource_id(BridgeId) when is_binary(BridgeId) -> invalid_data(<<"should be of pattern {type}:{name}, but got ", BridgeId/binary>>) end. -resource_id(BridgeType, BridgeName) -> - BridgeId = bridge_id(BridgeType, BridgeName), - resource_id(BridgeId). - bridge_id(BridgeType, BridgeName) -> Name = bin(BridgeName), Type = bin(BridgeType), diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index fb8ae2e2e..67aeeca41 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -91,6 +91,7 @@ id/2, id/3, bridge_v1_is_valid/2, + bridge_v1_is_valid/3, extract_connector_id_from_bridge_v2_id/1 ]). @@ -128,6 +129,7 @@ %% Exception from the naming convention: bridge_v2_type_to_bridge_v1_type/2, bridge_v1_id_to_connector_resource_id/1, + bridge_v1_id_to_connector_resource_id/2, bridge_v1_enable_disable/3, bridge_v1_restart/2, bridge_v1_stop/2, @@ -567,7 +569,7 @@ connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHe ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf(ConfRootKey, BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ). @@ -1191,8 +1193,11 @@ unpack_bridge_conf(Type, PackedConf, TopLevelConf) -> %% * The corresponding bridge v2 should exist %% * The connector for the bridge v2 should have exactly one channel bridge_v1_is_valid(BridgeV1Type, BridgeName) -> + bridge_v1_is_valid(?ROOT_KEY_ACTIONS, BridgeV1Type, BridgeName). + +bridge_v1_is_valid(ConfRootKey, BridgeV1Type, BridgeName) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf(ConfRootKey, BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 true; @@ -1241,17 +1246,20 @@ bridge_v1_list_and_transform() -> bridge_v1_lookup_and_transform(ActionType, Name) -> case lookup_actions_or_sources(ActionType, Name) of - {ok, ConfRootName, + {ok, ConfRootKey, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), HasBridgeV1Equivalent = has_bridge_v1_equivalent(ActionType), - case HasBridgeV1Equivalent andalso ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of + case + HasBridgeV1Equivalent andalso + ?MODULE:bridge_v1_is_valid(ConfRootKey, BridgeV1Type, Name) + of true -> ConnectorType = connector_type(ActionType), case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( - ConfRootName, + ConfRootKey, BridgeV1Type, Name, ActionType, @@ -1718,11 +1726,14 @@ connector_has_channels(BridgeV2Type, ConnectorName) -> end. bridge_v1_id_to_connector_resource_id(BridgeId) -> + bridge_v1_id_to_connector_resource_id(?ROOT_KEY_ACTIONS, BridgeId). + +bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, Name] -> BridgeV2Type = bin(bridge_v1_type_to_bridge_v2_type(Type)), ConnectorName = - case lookup_conf(BridgeV2Type, Name) of + case lookup_conf(ConfRootKey, BridgeV2Type, Name) of #{connector := Con} -> Con; {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 3f0d18fae..d4401cfd0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -37,7 +37,7 @@ namespace/0 ]). -%% API callbacks +%% API callbacks : actions -export([ '/actions'/2, '/actions/:id'/2, @@ -49,6 +49,18 @@ '/actions_probe'/2, '/action_types'/2 ]). +%% API callbacks : sources +-export([ + '/sources'/2, + '/sources/:id'/2, + '/sources/:id/metrics'/2, + '/sources/:id/metrics/reset'/2, + '/sources/:id/enable/:enable'/2, + '/sources/:id/:operation'/2, + '/nodes/:node/sources/:id/:operation'/2, + '/sources_probe'/2, + '/source_types'/2 +]). %% BpAPI / RPC Targets -export([ @@ -81,13 +93,16 @@ end ). -namespace() -> "actions". +namespace() -> "actions_and_sources". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ + %%============= + %% Actions + %%============= "/actions", "/actions/:id", "/actions/:id/enable/:enable", @@ -98,7 +113,21 @@ paths() -> "/actions/:id/metrics", "/actions/:id/metrics/reset", "/actions_probe", - "/action_types" + "/action_types", + %%============= + %% Sources + %%============= + "/sources", + "/sources/:id", + "/sources/:id/enable/:enable", + "/sources/:id/:operation", + "/nodes/:node/sources/:id/:operation", + %% %% Caveat: metrics paths must come *after* `/:operation', otherwise minirest will + %% %% try to match the latter first, trying to interpret `metrics' as an operation... + "/sources/:id/metrics", + "/sources/:id/metrics/reset", + "/sources_probe" + %% "/source_types" ]. error_schema(Code, Message) -> @@ -111,17 +140,28 @@ error_schema(Codes, Message, ExtraFields) when is_list(Message) -> error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) -> ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message). -get_response_body_schema() -> +actions_get_response_body_schema() -> emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:get_response(), - bridge_info_examples(get) + emqx_bridge_v2_schema:actions_get_response(), + bridge_info_examples(get, ?ROOT_KEY_ACTIONS) ). -bridge_info_examples(Method) -> - emqx_bridge_v2_schema:examples(Method). +sources_get_response_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_get_response(), + bridge_info_examples(get, ?ROOT_KEY_SOURCES) + ). -bridge_info_array_example(Method) -> - lists:map(fun(#{value := Config}) -> Config end, maps:values(bridge_info_examples(Method))). +bridge_info_examples(Method, ?ROOT_KEY_ACTIONS) -> + emqx_bridge_v2_schema:actions_examples(Method); +bridge_info_examples(Method, ?ROOT_KEY_SOURCES) -> + emqx_bridge_v2_schema:sources_examples(Method). + +bridge_info_array_example(Method, ConfRootKey) -> + lists:map( + fun(#{value := Config}) -> Config end, + maps:values(bridge_info_examples(Method, ConfRootKey)) + ). param_path_id() -> {id, @@ -195,6 +235,9 @@ param_path_enable() -> } )}. +%%================================================================================ +%% Actions +%%================================================================================ schema("/actions") -> #{ 'operationId' => '/actions', @@ -204,8 +247,8 @@ schema("/actions") -> description => ?DESC("desc_api1"), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - array(emqx_bridge_v2_schema:get_response()), - bridge_info_array_example(get) + array(emqx_bridge_v2_schema:actions_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_ACTIONS) ) } }, @@ -214,11 +257,11 @@ schema("/actions") -> summary => <<"Create bridge">>, description => ?DESC("desc_api2"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ - 201 => get_response_body_schema(), + 201 => actions_get_response_body_schema(), 400 => error_schema('ALREADY_EXISTS', "Bridge already exists") } } @@ -232,7 +275,7 @@ schema("/actions/:id") -> description => ?DESC("desc_api3"), parameters => [param_path_id()], responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found") } }, @@ -242,11 +285,11 @@ schema("/actions/:id") -> description => ?DESC("desc_api4"), parameters => [param_path_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:put_request(), - bridge_info_examples(put) + emqx_bridge_v2_schema:actions_put_request(), + bridge_info_examples(put, ?ROOT_KEY_ACTIONS) ), responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found"), 400 => error_schema('BAD_REQUEST', "Update bridge failed") } @@ -371,8 +414,8 @@ schema("/actions_probe") -> desc => ?DESC("desc_api9"), summary => <<"Test creating bridge">>, 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ 204 => <<"Test bridge OK">>, @@ -389,12 +432,223 @@ schema("/action_types") -> summary => <<"List available action types">>, responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - array(emqx_bridge_v2_schema:types_sc()), + array(emqx_bridge_v2_schema:action_types_sc()), #{ <<"types">> => #{ summary => <<"Action types">>, - value => emqx_bridge_v2_schema:types() + value => emqx_bridge_v2_schema:action_types() + } + } + ) + } + } + }; +%%================================================================================ +%% Sources +%%================================================================================ +schema("/sources") -> + #{ + 'operationId' => '/sources', + get => #{ + tags => [<<"sources">>], + summary => <<"List sources">>, + description => ?DESC("desc_api1"), + responses => #{ + %% FIXME: examples + 200 => emqx_dashboard_swagger:schema_with_example( + array(emqx_bridge_v2_schema:sources_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_SOURCES) + ) + } + }, + post => #{ + tags => [<<"sources">>], + summary => <<"Create source">>, + description => ?DESC("desc_api2"), + %% FIXME: examples + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 201 => sources_get_response_body_schema(), + 400 => error_schema('ALREADY_EXISTS', "Source already exists") + } + } + }; +schema("/sources/:id") -> + #{ + 'operationId' => '/sources/:id', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source">>, + description => ?DESC("desc_api3"), + parameters => [param_path_id()], + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + }, + put => #{ + tags => [<<"sources">>], + summary => <<"Update source">>, + description => ?DESC("desc_api4"), + parameters => [param_path_id()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_put_request(), + bridge_info_examples(put, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found"), + 400 => error_schema('BAD_REQUEST', "Update source failed") + } + }, + delete => #{ + tags => [<<"sources">>], + summary => <<"Delete source">>, + description => ?DESC("desc_api5"), + parameters => [param_path_id(), param_qs_delete_cascade()], + responses => #{ + 204 => <<"Source deleted">>, + 400 => error_schema( + 'BAD_REQUEST', + "Cannot delete bridge while active rules are defined for this source", + [{rules, mk(array(string()), #{desc => "Dependent Rule IDs"})}] + ), + 404 => error_schema('NOT_FOUND', "Source not found"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/metrics") -> + #{ + 'operationId' => '/sources/:id/metrics', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source metrics">>, + description => ?DESC("desc_bridge_metrics"), + parameters => [param_path_id()], + responses => #{ + 200 => emqx_bridge_schema:metrics_fields(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/metrics/reset") -> + #{ + 'operationId' => '/sources/:id/metrics/reset', + put => #{ + tags => [<<"sources">>], + summary => <<"Reset source metrics">>, + description => ?DESC("desc_api6"), + parameters => [param_path_id()], + responses => #{ + 204 => <<"Reset success">>, + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/enable/:enable") -> + #{ + 'operationId' => '/sources/:id/enable/:enable', + put => + #{ + tags => [<<"sources">>], + summary => <<"Enable or disable bridge">>, + desc => ?DESC("desc_enable_bridge"), + parameters => [param_path_id(), param_path_enable()], + responses => + #{ + 204 => <<"Success">>, + 404 => error_schema( + 'NOT_FOUND', "Bridge not found or invalid operation" + ), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/:operation") -> + #{ + 'operationId' => '/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge">>, + description => ?DESC("desc_api7"), + parameters => [ + param_path_id(), + param_path_operation_cluster() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', "Problem with configuration of external service" + ), + 404 => error_schema('NOT_FOUND', "Bridge not found or invalid operation"), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/nodes/:node/sources/:id/:operation") -> + #{ + 'operationId' => '/nodes/:node/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge on a given node">>, + description => ?DESC("desc_api8"), + parameters => [ + param_path_node(), + param_path_id(), + param_path_operation_on_node() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', + "Problem with configuration of external service or bridge not enabled" + ), + 404 => error_schema( + 'NOT_FOUND', "Bridge or node not found or invalid operation" + ), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources_probe") -> + #{ + 'operationId' => '/sources_probe', + post => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api9"), + summary => <<"Test creating bridge">>, + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 204 => <<"Test bridge OK">>, + 400 => error_schema(['TEST_FAILED'], "bridge test failed") + } + } + }; +schema("/source_types") -> + #{ + 'operationId' => '/source_types', + get => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api10"), + summary => <<"List available source types">>, + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + array(emqx_bridge_v2_schema:action_types_sc()), + #{ + <<"types">> => + #{ + summary => <<"Source types">>, + value => emqx_bridge_v2_schema:action_types() } } ) @@ -402,6 +656,12 @@ schema("/action_types") -> } }. +%%------------------------------------------------------------------------------ +%% Thin Handlers +%%------------------------------------------------------------------------------ +%%================================================================================ +%% Actions +%%================================================================================ '/actions'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> handle_create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, Conf0); '/actions'(get, _Params) -> @@ -439,7 +699,48 @@ schema("/action_types") -> handle_probe(?ROOT_KEY_ACTIONS, Request). '/action_types'(get, _Request) -> - ?OK(emqx_bridge_v2_schema:types()). + ?OK(emqx_bridge_v2_schema:action_types()). +%%================================================================================ +%% Sources +%%================================================================================ +'/sources'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> + handle_create(?ROOT_KEY_SOURCES, BridgeType, BridgeName, Conf0); +'/sources'(get, _Params) -> + handle_list(?ROOT_KEY_SOURCES). + +'/sources/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName, 200)); +'/sources/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_SOURCES, Id, Conf0); +'/sources/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_SOURCES, Id, Qs). + +'/sources/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName)). + +'/sources/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_SOURCES, Id). + +'/sources/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_SOURCES, Id, Enable). + +'/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_SOURCES, Id, Op). + +'/nodes/:node/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_SOURCES, Node, Id, Op). + +'/sources_probe'(post, Request) -> + handle_probe(?ROOT_KEY_SOURCES, Request). + +'/source_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:source_types()). %%------------------------------------------------------------------------------ %% Handlers @@ -451,7 +752,7 @@ handle_list(ConfRootKey) -> case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ - [format_resource(Data, Node) || Data <- Bridges] + [format_resource(ConfRootKey, Data, Node) || Data <- Bridges] || {Node, Bridges} <- lists:zip(Nodes, NodeBridges) ], ?OK(zip_bridges(AllBridges)); @@ -574,7 +875,12 @@ handle_node_operation(ConfRootKey, Node, Id, Op) -> ). handle_probe(ConfRootKey, Request) -> - RequestMeta = #{module => ?MODULE, method => post, path => "/actions_probe"}, + Path = + case ConfRootKey of + ?ROOT_KEY_ACTIONS -> "/actions_probe"; + ?ROOT_KEY_SOURCES -> "/sources_probe" + end, + RequestMeta = #{module => ?MODULE, method => post, path => Path}, case emqx_dashboard_swagger:filter_check_request_and_translate_body(Request, RequestMeta) of {ok, #{body := #{<<"type">> := Type} = Params}} -> Params1 = maybe_deobfuscate_bridge_probe(Params), @@ -664,8 +970,8 @@ get_metrics_from_all_nodes(ConfRootKey, Type, Name) -> ?INTERNAL_ERROR(Reason) end. -operation_func(all, start) -> v2_start_bridge_to_all_nodes_v6; -operation_func(_Node, start) -> v2_start_bridge_to_node_v6; +operation_func(all, start) -> v2_start_bridge_on_all_nodes_v6; +operation_func(_Node, start) -> v2_start_bridge_on_node_v6; operation_func(all, lookup) -> v2_lookup_from_all_nodes_v6; operation_func(all, list) -> v2_list_bridges_on_nodes_v6; operation_func(all, get_metrics) -> v2_get_metrics_from_all_nodes_v6. @@ -825,7 +1131,7 @@ aggregate_status(AllStatus) -> %% RPC Target lookup_from_local_node(BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resource(Res, node())}; + {ok, Res} -> {ok, format_resource(?ROOT_KEY_ACTIONS, Res, node())}; Error -> Error end. @@ -833,7 +1139,7 @@ lookup_from_local_node(BridgeType, BridgeName) -> -spec lookup_from_local_node_v6(emqx_bridge_v2:root_cfg_key(), _, _) -> _. lookup_from_local_node_v6(ConfRootKey, BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resource(Res, node())}; + {ok, Res} -> {ok, format_resource(ConfRootKey, Res, node())}; Error -> Error end. @@ -847,6 +1153,7 @@ get_metrics_from_local_node_v6(ConfRootKey, Type, Name) -> %% resource format_resource( + ConfRootKey, #{ type := Type, name := Name, @@ -857,7 +1164,7 @@ format_resource( }, Node ) -> - RawConf = fill_defaults(Type, RawConf0), + RawConf = fill_defaults(ConfRootKey, Type, RawConf0), redact( maps:merge( RawConf#{ @@ -988,17 +1295,18 @@ aggregate_metrics( M17 + N17 ). -fill_defaults(Type, RawConf) -> - PackedConf = pack_bridge_conf(Type, RawConf), +fill_defaults(ConfRootKey, Type, RawConf) -> + PackedConf = pack_bridge_conf(ConfRootKey, Type, RawConf), FullConf = emqx_config:fill_defaults(emqx_bridge_v2_schema, PackedConf, #{}), - unpack_bridge_conf(Type, FullConf). + unpack_bridge_conf(ConfRootKey, Type, FullConf). -pack_bridge_conf(Type, RawConf) -> - #{<<"actions">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. +pack_bridge_conf(ConfRootKey, Type, RawConf) -> + #{bin(ConfRootKey) => #{bin(Type) => #{<<"foo">> => RawConf}}}. -unpack_bridge_conf(Type, PackedConf) -> +unpack_bridge_conf(ConfRootKey, Type, PackedConf) -> + ConfRootKeyBin = bin(ConfRootKey), TypeBin = bin(Type), - #{<<"actions">> := Bridges} = PackedConf, + #{ConfRootKeyBin := Bridges} = PackedConf, #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), RawConf. diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl index d6fe68466..fbcef8b5c 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl @@ -35,8 +35,8 @@ v2_lookup_from_all_nodes_v6/4, v2_list_bridges_on_nodes_v6/2, v2_get_metrics_from_all_nodes_v6/4, - v2_start_bridge_to_node_v6/4, - v2_start_bridge_to_all_nodes_v6/4 + v2_start_bridge_on_node_v6/4, + v2_start_bridge_on_all_nodes_v6/4 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -173,9 +173,9 @@ v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, ActionType, ActionName) -> ?TIMEOUT ). --spec v2_start_bridge_to_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> +-spec v2_start_bridge_on_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> emqx_rpc:erpc_multicall(ok). -v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> +v2_start_bridge_on_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> erpc:multicall( Nodes, emqx_bridge_v2, @@ -184,9 +184,9 @@ v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> ?TIMEOUT ). --spec v2_start_bridge_to_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> +-spec v2_start_bridge_on_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> term(). -v2_start_bridge_to_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> +v2_start_bridge_on_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> rpc:call( Node, emqx_bridge_v2, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 28017f814..ec9314fd2 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -28,21 +28,31 @@ -export([roots/0, fields/1, desc/1, namespace/0, tags/0]). -export([ - get_response/0, - put_request/0, - post_request/0, - examples/1, + actions_get_response/0, + actions_put_request/0, + actions_post_request/0, + actions_examples/1, action_values/4 ]). +-export([ + sources_get_response/0, + sources_put_request/0, + sources_post_request/0, + sources_examples/1, + source_values/4 +]). + %% Exported for mocking %% TODO: refactor emqx_bridge_v1_compatibility_layer_SUITE so we don't need to %% export this -export([ - registered_api_schemas/1 + registered_actions_api_schemas/1, + registered_sources_api_schemas/1 ]). --export([types/0, types_sc/0]). +-export([action_types/0, action_types_sc/0]). +-export([source_types/0, source_types_sc/0]). -export([resource_opts_fields/0, resource_opts_fields/1]). -export([ @@ -58,33 +68,140 @@ -export([actions_convert_from_connectors/1]). --export_type([action_type/0]). +-export_type([action_type/0, source_type/0]). %% Should we explicitly list them here so dialyzer may be more helpful? -type action_type() :: atom(). +-type source_type() :: atom(). +-type http_method() :: get | post | put. +-type schema_example_map() :: #{atom() => term()}. %%====================================================================================== %% For HTTP APIs -get_response() -> - api_schema("get"). +%%====================================================================================== -put_request() -> - api_schema("put"). +%%--------------------------------------------- +%% Actions +%%--------------------------------------------- -post_request() -> - api_schema("post"). +actions_get_response() -> + actions_api_schema("get"). -api_schema(Method) -> - APISchemas = ?MODULE:registered_api_schemas(Method), +actions_put_request() -> + actions_api_schema("put"). + +actions_post_request() -> + actions_api_schema("post"). + +actions_api_schema(Method) -> + APISchemas = ?MODULE:registered_actions_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). -registered_api_schemas(Method) -> +registered_actions_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_bridge_v2") || {BridgeV2Type, SchemaModule} <- RegisteredSchemas ]. +-spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +action_values(Method, ActionType, ConnectorType, ActionValues) -> + ActionTypeBin = atom_to_binary(ActionType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", ActionTypeBin/binary, " action">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + ActionValues, + method_values(action, Method, ActionType) + ] + ). + +actions_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Sources +%%--------------------------------------------- + +sources_get_response() -> + sources_api_schema("get"). + +sources_put_request() -> + sources_api_schema("put"). + +sources_post_request() -> + sources_api_schema("post"). + +sources_api_schema(Method) -> + APISchemas = ?MODULE:registered_sources_api_schemas(Method), + hoconsc:union(bridge_api_union(APISchemas)). + +registered_sources_api_schemas(Method) -> + RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(), + [ + api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_source") + || {BridgeV2Type, SchemaModule} <- RegisteredSchemas + ]. + +-spec source_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +source_values(Method, SourceType, ConnectorType, SourceValues) -> + SourceTypeBin = atom_to_binary(SourceType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", SourceTypeBin/binary, " source">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + SourceValues, + method_values(source, Method, SourceType) + ] + ). + +sources_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_sources()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Common helpers +%%--------------------------------------------- + api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. @@ -111,41 +228,17 @@ bridge_api_union(Refs) -> end end. --type http_method() :: get | post | put. --type schema_example_map() :: #{atom() => term()}. - --spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). -action_values(Method, ActionType, ConnectorType, ActionValues) -> - ActionTypeBin = atom_to_binary(ActionType), - ConnectorTypeBin = atom_to_binary(ConnectorType), - lists:foldl( - fun(M1, M2) -> - maps:merge(M1, M2) - end, - #{ - enable => true, - description => <<"My example ", ActionTypeBin/binary, " action">>, - connector => <>, - resource_opts => #{ - health_check_interval => "30s" - } - }, - [ - ActionValues, - method_values(Method, ActionType) - ] - ). - --spec method_values(http_method(), atom()) -> schema_example_map(). -method_values(post, Type) -> +-spec method_values(action | source, http_method(), atom()) -> schema_example_map(). +method_values(Kind, post, Type) -> + KindBin = atom_to_binary(Kind), TypeBin = atom_to_binary(Type), #{ - name => <>, + name => <>, type => TypeBin }; -method_values(get, Type) -> +method_values(Kind, get, Type) -> maps:merge( - method_values(post, Type), + method_values(Kind, post, Type), #{ status => <<"connected">>, node_status => [ @@ -156,7 +249,7 @@ method_values(get, Type) -> ] } ); -method_values(put, _Type) -> +method_values(_Kind, put, _Type) -> #{}. api_fields("get_bridge_v2", Type, Fields) -> @@ -175,16 +268,33 @@ api_fields("post_bridge_v2", Type, Fields) -> ] ); api_fields("put_bridge_v2", _Type, Fields) -> + Fields; +api_fields("get_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + emqx_bridge_schema:status_fields(), + Fields + ] + ); +api_fields("post_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + Fields + ] + ); +api_fields("put_source", _Type, Fields) -> Fields. %%====================================================================================== %% HOCON Schema Callbacks %%====================================================================================== -namespace() -> "actions". +namespace() -> "actions_and_sources". tags() -> - [<<"Actions">>]. + [<<"Actions">>, <<"Sources">>]. -dialyzer({nowarn_function, roots/0}). @@ -231,13 +341,21 @@ desc(resource_opts) -> desc(_) -> undefined. --spec types() -> [action_type()]. -types() -> +-spec action_types() -> [action_type()]. +action_types() -> proplists:get_keys(?MODULE:fields(actions)). --spec types_sc() -> ?ENUM([action_type()]). -types_sc() -> - hoconsc:enum(types()). +-spec action_types_sc() -> ?ENUM([action_type()]). +action_types_sc() -> + hoconsc:enum(action_types()). + +-spec source_types() -> [source_type()]. +source_types() -> + proplists:get_keys(?MODULE:fields(sources)). + +-spec source_types_sc() -> ?ENUM([source_type()]). +source_types_sc() -> + hoconsc:enum(source_types()). resource_opts_fields() -> resource_opts_fields(_Overrides = []). @@ -268,19 +386,6 @@ resource_opts_fields(Overrides) -> emqx_resource_schema:create_opts(Overrides) ). -examples(Method) -> - MergeFun = - fun(Example, Examples) -> - maps:merge(Examples, Example) - end, - Fun = - fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), - lists:foldl(MergeFun, Examples, ConnectorExamples) - end, - SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], - lists:foldl(Fun, #{}, SchemaModules). - top_level_common_action_keys() -> [ <<"connector">>, diff --git a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl index dadc0a09c..b67791cb3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl @@ -104,7 +104,7 @@ setup_mocks() -> catch meck:new(emqx_bridge_v2_schema, MeckOpts), meck:expect( emqx_bridge_v2_schema, - registered_api_schemas, + registered_actions_api_schemas, 1, fun(Method) -> [{bridge_type_bin(), hoconsc:ref(?MODULE, "api_v2_" ++ Method)}] diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index f7dd74161..88788d6e2 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -12,6 +12,9 @@ -import(emqx_common_test_helpers, [on_exit/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% ct setup helpers init_per_suite(Config, Apps) -> @@ -152,6 +155,49 @@ create_bridge(Config, Overrides) -> ct:pal("creating bridge with config: ~p", [BridgeConfig]), emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig). +get_ct_config_with_fallback(Config, [Key]) -> + ?config(Key, Config); +get_ct_config_with_fallback(Config, [Key | Rest]) -> + case ?config(Key, Config) of + undefined -> + get_ct_config_with_fallback(Config, Rest); + X -> + X + end. + +get_config_by_kind(Config, Overrides) -> + Kind = ?config(bridge_kind, Config), + get_config_by_kind(Kind, Config, Overrides). + +get_config_by_kind(Kind, Config, Overrides) -> + case Kind of + action -> + %% TODO: refactor tests to use action_type... + ActionType = get_ct_config_with_fallback(Config, [action_type, bridge_type]), + ActionName = get_ct_config_with_fallback(Config, [action_name, bridge_name]), + ActionConfig0 = get_ct_config_with_fallback(Config, [action_config, bridge_config]), + ActionConfig = emqx_utils_maps:deep_merge(ActionConfig0, Overrides), + #{type => ActionType, name => ActionName, config => ActionConfig}; + source -> + SourceType = ?config(source_type, Config), + SourceName = ?config(source_name, Config), + SourceConfig0 = ?config(source_config, Config), + SourceConfig = emqx_utils_maps:deep_merge(SourceConfig0, Overrides), + #{type => SourceType, name => SourceName, config => SourceConfig} + end. + +api_path_root(Kind) -> + case Kind of + action -> "actions"; + source -> "sources" + end. + +conf_root_key(Kind) -> + case Kind of + action -> ?ROOT_KEY_ACTIONS; + source -> ?ROOT_KEY_SOURCES + end. + maybe_json_decode(X) -> case emqx_utils_json:safe_decode(X, [return_maps]) of {ok, Decoded} -> Decoded; @@ -218,26 +264,26 @@ create_bridge_api(Config) -> create_bridge_api(Config, _Overrides = #{}). create_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - {ok, {{_, 201, _}, _, _}} = create_connector_api(Config), + create_kind_api(Config, Overrides). - Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("creating bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge create result: ~p", [Res]), +create_kind_api(Config) -> + create_kind_api(Config, _Overrides = #{}). + +create_kind_api(Config, Overrides) -> + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, Overrides), + Params = BridgeConfig#{<<"type">> => Type, <<"name">> => Name}, + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot]), + ct:pal("creating bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge create (~s, http) result:\n ~p", [Kind, Res]), Res. create_connector_api(Config) -> @@ -288,27 +334,29 @@ update_bridge_api(Config) -> update_bridge_api(Config, _Overrides = #{}). update_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - Name = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - BridgeId = emqx_bridge_resource:bridge_id(BridgeType, Name), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("updating bridge (via http): ~p", [BridgeConfig]), - Res = - case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, BridgeConfig, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; - Error -> Error - end, - ct:pal("bridge update result: ~p", [Res]), + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := Params + } = get_config_by_kind(Kind, Config, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId]), + ct:pal("updating bridge (~s, http):\n ~p", [Kind, Params]), + Method = put, + Res = request(Method, Path, Params), + ct:pal("update bridge (~s, http) result:\n ~p", [Kind, Res]), Res. op_bridge_api(Op, BridgeType, BridgeName) -> + op_bridge_api(_Kind = action, Op, BridgeType, BridgeName). + +op_bridge_api(Kind, Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId, Op]), - ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId, Op]), + ct:pal("calling bridge ~p (~s, http):\n ~p", [BridgeId, Kind, Op]), Method = post, Params = [], Res = request(Method, Path, Params), @@ -326,17 +374,16 @@ probe_bridge_api(Config, Overrides) -> probe_bridge_api(BridgeType, BridgeName, BridgeConfig). probe_bridge_api(BridgeType, BridgeName, BridgeConfig) -> + probe_bridge_api(action, BridgeType, BridgeName, BridgeConfig). + +probe_bridge_api(Kind, BridgeType, BridgeName, BridgeConfig) -> Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions_probe"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("probing bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; - Error -> Error - end, - ct:pal("bridge probe result: ~p", [Res]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot ++ "_probe"]), + ct:pal("probing bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge probe (~s, http) result:\n ~p", [Kind, Res]), Res. list_bridges_http_api_v1() -> @@ -353,6 +400,13 @@ list_actions_http_api() -> ct:pal("list actions (http v2) result:\n ~p", [Res]), Res. +list_sources_http_api() -> + Path = emqx_mgmt_api_test_util:api_path(["sources"]), + ct:pal("list sources (http v2)"), + Res = request(get, Path, _Params = []), + ct:pal("list sources (http v2) result:\n ~p", [Res]), + Res. + list_connectors_http_api() -> Path = emqx_mgmt_api_test_util:api_path(["connectors"]), ct:pal("list connectors"), @@ -506,13 +560,6 @@ t_create_via_http(Config) -> begin ?assertMatch({ok, _}, create_bridge_api(Config)), - %% lightweight matrix testing some configs - ?assertMatch( - {ok, _}, - update_bridge_api( - Config - ) - ), ?assertMatch( {ok, _}, update_bridge_api( @@ -526,23 +573,26 @@ t_create_via_http(Config) -> ok. t_start_stop(Config, StopTracePoint) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig = ?config(bridge_config, Config), + Kind = proplists:get_value(bridge_kind, Config, action), ConnectorName = ?config(connector_name, Config), ConnectorType = ?config(connector_type, Config), - ConnectorConfig = ?config(connector_config, Config), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, _Overrides = #{}), ?assertMatch( - {ok, _}, - emqx_connector:create(ConnectorType, ConnectorName, ConnectorConfig) + {ok, {{_, 201, _}, _, _}}, + create_connector_api(Config) ), ?check_trace( begin ProbeRes0 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), @@ -550,8 +600,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsBefore = erlang:system_info(atom_count), %% Probe again; shouldn't have created more atoms. ProbeRes1 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), @@ -559,9 +610,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), - ?assertMatch({ok, _}, emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig)), + ?assertMatch({ok, _}, create_kind_api(Config)), - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + ResourceId = emqx_bridge_resource:resource_id(conf_root_key(Kind), Type, Name), %% Since the connection process is async, we give it some time to %% stabilize and avoid flakiness. @@ -574,7 +625,7 @@ t_start_stop(Config, StopTracePoint) -> %% `start` bridge to trigger `already_started` ?assertMatch( {ok, {{_, 204, _}, _Headers, []}}, - emqx_bridge_v2_testlib:op_bridge_api("start", BridgeType, BridgeName) + op_bridge_api(Kind, "start", Type, Name) ), ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), @@ -624,10 +675,10 @@ t_start_stop(Config, StopTracePoint) -> ) ), - ok + #{resource_id => ResourceId} end, - fun(Trace) -> - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + fun(Res, Trace) -> + #{resource_id := ResourceId} = Res, %% one for each probe, one for real ?assertMatch( [_, _, #{instance_id := ResourceId}], diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index e90100995..c64b1f2cb 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -108,7 +108,7 @@ connector_resource_opts_test() -> ok. actions_api_spec_post_fields_test() -> - ?UNION(Union) = emqx_bridge_v2_schema:post_request(), + ?UNION(Union) = emqx_bridge_v2_schema:actions_post_request(), Schemas = lists:map( fun(?R_REF(SchemaMod, StructName)) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index dbdf68ef1..18af6ee11 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -237,7 +237,9 @@ on_stop(ResourceId, State) -> ets:delete(TopicToHandlerIndex) end, Allocated = emqx_resource:get_allocated_resources(ResourceId), - ok = stop_helper(Allocated). + ok = stop_helper(Allocated), + ?tp(mqtt_connector_stopped, #{instance_id => ResourceId}), + ok. stop_helper(#{pool_name := PoolName}) -> emqx_resource_pool:stop(PoolName). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 6d075334a..f765581f9 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -27,6 +27,9 @@ conn_bridge_examples/1 ]). +-define(ACTION_TYPE, mqtt). +-define(SOURCE_TYPE, mqtt). + %%====================================================================================== %% Hocon Schema Definitions namespace() -> "bridge_mqtt_publisher". @@ -86,14 +89,18 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, emqx_bridge_v2_schema:resource_opts_fields() ); -fields("get_connector") -> - emqx_bridge_mqtt_connector_schema:fields("config_connector"); -fields("get_bridge_v2") -> - fields("mqtt_publisher_action"); -fields("post_bridge_v2") -> - fields("mqtt_publisher_action") ++ emqx_bridge_schema:type_and_name_fields(mqtt); -fields("put_bridge_v2") -> - fields("mqtt_publisher_action"); +fields(Field) when + Field == "get_bridge_v2"; + Field == "post_bridge_v2"; + Field == "put_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION_TYPE, fields("mqtt_publisher_action")); +fields(Field) when + Field == "get_source"; + Field == "post_source"; + Field == "put_source" +-> + emqx_bridge_v2_schema:api_fields(Field, ?SOURCE_TYPE, fields("mqtt_subscriber_source")); fields(What) -> error({emqx_bridge_mqtt_pubsub_schema, missing_field_handler, What}). %% v2: api schema diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl new file mode 100644 index 000000000..fde15a1b6 --- /dev/null +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -0,0 +1,175 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_v2_subscriber_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, Api} = emqx_common_test_http:create_default_app(), + [ + {apps, Apps}, + {api, Api} + | Config + ]. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), + ok. + +init_per_testcase(TestCase, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + Name = iolist_to_binary([atom_to_binary(TestCase), UniqueNum]), + ConnectorConfig = connector_config(), + SourceConfig = source_config(#{connector => Name}), + [ + {bridge_kind, source}, + {source_type, mqtt}, + {source_name, Name}, + {source_config, SourceConfig}, + {connector_type, mqtt}, + {connector_name, Name}, + {connector_config, ConnectorConfig} + | Config + ]. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +connector_config() -> + %% !!!!!!!!!!!! FIXME!!!!!! add more fields ("server_configs") + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + CommonConfig = + #{ + <<"enable">> => true, + <<"connector">> => <<"please override">>, + <<"parameters">> => + #{ + <<"remote">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + }, + <<"local">> => + #{ + <<"topic">> => <<"local/topic">>, + <<"qos">> => 2, + <<"retain">> => false, + <<"payload">> => <<"${payload}">> + } + }, + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"batch_time">> => <<"0ms">>, + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"15s">>, + <<"inflight_window">> => 100, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"45s">>, + <<"resume_interval">> => <<"15s">>, + <<"worker_pool_size">> => <<"1">> + } + }, + maps:merge(CommonConfig, Overrides). + +replace(Key, Value, Proplist) -> + lists:keyreplace(Key, 1, Proplist, {Key, Value}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_create_via_http(Config) -> + ConnectorName = ?config(connector_name, Config), + ok = emqx_bridge_v2_testlib:t_create_via_http(Config), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"enable">> := true, + <<"status">> := <<"connected">> + } + ]}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + NewSourceName = <<"my_other_source">>, + {ok, {{_, 201, _}, _, _}} = + emqx_bridge_v2_testlib:create_kind_api( + replace(source_name, NewSourceName, Config) + ), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{<<"connector">> := ConnectorName}, + #{<<"connector">> := ConnectorName} + ]}}, + emqx_bridge_v2_testlib:list_sources_http_api() + ), + ?assertMatch( + {ok, {{_, 200, _}, _, []}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + ok. + +t_start_stop(Config) -> + ok = emqx_bridge_v2_testlib:t_start_stop(Config, mqtt_connector_stopped), + ok. From cc24fe6e933dbd563a3ddd7273cb09dda32396f8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Jan 2024 15:12:10 -0300 Subject: [PATCH 19/28] feat(mqtt_consumer): add support for rule engine `FROM` --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 13 ++- .../test/emqx_bridge_v2_testlib.erl | 17 ++++ .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 3 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 96 +++++++++++++++++-- .../emqx_connector/src/emqx_connector_api.erl | 7 +- .../src/emqx_rule_actions.erl | 6 ++ .../emqx_rule_engine/src/emqx_rule_engine.erl | 9 +- 8 files changed, 135 insertions(+), 18 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 67aeeca41..32034f774 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -54,6 +54,7 @@ check_deps_and_remove/3, check_deps_and_remove/4 ]). +-export([lookup_action/2, lookup_source/2]). %% Operations @@ -222,6 +223,12 @@ unload_bridges(ConfRooKey) -> lookup(Type, Name) -> lookup(?ROOT_KEY_ACTIONS, Type, Name). +lookup_action(Type, Name) -> + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_source(Type, Name) -> + lookup(?ROOT_KEY_SOURCES, Type, Name). + -spec lookup(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(ConfRootName, Type, Name) -> @@ -900,9 +907,11 @@ do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) -> parse_id(Id) -> case binary:split(Id, <<":">>, [global]) of [Type, Name] -> - {Type, Name}; + #{kind => undefined, type => Type, name => Name}; [<<"action">>, Type, Name | _] -> - {Type, Name}; + #{kind => action, type => Type, name => Name}; + [<<"source">>, Type, Name | _] -> + #{kind => source, type => Type, name => Name}; _X -> error({error, iolist_to_binary(io_lib:format("Invalid id: ~p", [Id]))}) end. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 88788d6e2..7fef33115 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -442,6 +442,23 @@ try_decode_error(Body0) -> Body0 end. +create_rule_api(Opts) -> + #{ + sql := SQL, + actions := RuleActions + } = Opts, + Params = #{ + enable => true, + sql => SQL, + actions => RuleActions + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + ct:pal("create rule:\n ~p", [Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("create rule results:\n ~p", [Res]), + Res. + create_rule_and_action_http(BridgeType, RuleTopic, Config) -> create_rule_and_action_http(BridgeType, RuleTopic, Config, _Opts = #{}). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 7caab1d87..459e259d2 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -135,7 +135,7 @@ create_producers_for_bridge_v2( KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), - {_BridgeType, BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), + #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), IsDryRun = case TestIdStart of diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index f765581f9..b4c2b63ba 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -82,6 +82,7 @@ fields("mqtt_subscriber_source") -> fields(ingress_parameters) -> Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), Fields1 = proplists:delete(pool_size, Fields0), + %% FIXME: should we make `local` hidden? Fields1; fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], @@ -120,8 +121,6 @@ desc(ingress_parameters) -> ?DESC(ingress_parameters); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; -desc("config_connector") -> - ?DESC("desc_config"); desc("http_action") -> ?DESC("desc_config"); desc("parameters_opts") -> diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index fde15a1b6..5569a826b 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -18,6 +18,7 @@ -compile(export_all). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("emqx/include/asserts.hrl"). @@ -75,6 +76,11 @@ init_per_testcase(TestCase, Config) -> | Config ]. +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), + ok. + %%------------------------------------------------------------------------------ %% Helper fns %%------------------------------------------------------------------------------ @@ -85,6 +91,7 @@ connector_config() -> <<"enable">> => true, <<"description">> => <<"my connector">>, <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, <<"server">> => <<"127.0.0.1:1883">>, <<"resource_opts">> => #{ <<"health_check_interval">> => <<"15s">>, @@ -105,13 +112,6 @@ source_config(Overrides0) -> #{ <<"topic">> => <<"remote/topic">>, <<"qos">> => 2 - }, - <<"local">> => - #{ - <<"topic">> => <<"local/topic">>, - <<"qos">> => 2, - <<"retain">> => false, - <<"payload">> => <<"${payload}">> } }, <<"resource_opts">> => #{ @@ -134,6 +134,15 @@ source_config(Overrides0) -> replace(Key, Value, Proplist) -> lists:keyreplace(Key, 1, Proplist, {Key, Value}). +bridge_id(Config) -> + Type = ?config(source_type, Config), + Name = ?config(source_name, Config), + emqx_bridge_resource:bridge_id(Type, Name). + +hookpoint(Config) -> + BridgeId = bridge_id(Config), + emqx_bridge_resource:bridge_hookpoint(BridgeId). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -151,6 +160,11 @@ t_create_via_http(Config) -> ]}}, emqx_bridge_v2_testlib:list_bridges_http_api_v1() ), + ?assertMatch( + {ok, {{_, 200, _}, _, [#{<<"enable">> := true}]}}, + emqx_bridge_v2_testlib:list_connectors_http_api() + ), + NewSourceName = <<"my_other_source">>, {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_kind_api( @@ -173,3 +187,71 @@ t_create_via_http(Config) -> t_start_stop(Config) -> ok = emqx_bridge_v2_testlib:t_start_stop(Config, mqtt_connector_stopped), ok. + +t_receive_via_rule(Config) -> + SourceConfig = ?config(source_config, Config), + ?check_trace( + begin + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_connector_api(Config), + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_kind_api(Config), + Hookpoint = hookpoint(Config), + RepublishTopic = <<"rep/t">>, + RemoteTopic = emqx_utils_maps:deep_get( + [<<"parameters">>, <<"remote">>, <<"topic">>], + SourceConfig + ), + RuleOpts = #{ + sql => <<"select * from \"", Hookpoint/binary, "\"">>, + actions => [ + %% #{function => console}, + #{ + function => republish, + args => #{ + topic => RepublishTopic, + payload => <<"${.}">>, + qos => 0, + retain => false, + user_properties => <<"${.pub_props.'User-Property'}">> + } + } + ] + }, + {ok, {{_, 201, _}, _, #{<<"id">> := RuleId}}} = + emqx_bridge_v2_testlib:create_rule_api(RuleOpts), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), + {ok, Client} = emqtt:start_link([{proto_ver, v5}]), + {ok, _} = emqtt:connect(Client), + {ok, _, [?RC_GRANTED_QOS_0]} = emqtt:subscribe(Client, RepublishTopic), + ok = emqtt:publish( + Client, + RemoteTopic, + #{'User-Property' => [{<<"key">>, <<"value">>}]}, + <<"mypayload">>, + _Opts = [] + ), + {publish, Msg} = + ?assertReceive( + {publish, #{ + topic := RepublishTopic, + retain := false, + qos := 0, + properties := #{'User-Property' := [{<<"key">>, <<"value">>}]} + }} + ), + Payload = emqx_utils_json:decode(maps:get(payload, Msg), [return_maps]), + ?assertMatch( + #{ + <<"event">> := Hookpoint, + <<"payload">> := <<"mypayload">> + }, + Payload + ), + emqtt:stop(Client), + ok + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("action_references_nonexistent_bridges", Trace)), + ok + end + ), + ok. diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 83d387cd7..aae913001 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -326,7 +326,7 @@ schema("/connectors_probe") -> create_connector(ConnectorType, ConnectorName, Conf) end; '/connectors'(get, _Params) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), NodeReplies = emqx_connector_proto_v1:list_connectors_on_nodes(Nodes), case is_ok(NodeReplies) of {ok, NodeConnectors} -> @@ -674,7 +674,10 @@ unpack_connector_conf(Type, PackedConf) -> RawConf. format_action(ActionId) -> - element(2, emqx_bridge_v2:parse_id(ActionId)). + case emqx_bridge_v2:parse_id(ActionId) of + #{name := Name} -> + Name + end. is_ok(ok) -> ok; diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index cd8d597de..30e60df2b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -241,6 +241,12 @@ parse_user_properties(<<"${pub_props.'User-Property'}">>) -> %% we do not want to force users to select the value %% the value will be taken from Env.pub_props directly ?ORIGINAL_USER_PROPERTIES; +parse_user_properties(<<"${.pub_props.'User-Property'}">>) -> + %% keep the original + %% avoid processing this special variable because + %% we do not want to force users to select the value + %% the value will be taken from Env.pub_props directly + ?ORIGINAL_USER_PROPERTIES; parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable emqx_template:parse(V); diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 70a7fc32c..b58d877e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -23,6 +23,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([start_link/0]). @@ -482,8 +483,7 @@ with_parsed_rule(Params = #{id := RuleId, sql := Sql, actions := Actions}, Creat ok -> ok; {error, NonExistentBridgeIDs} -> - ?SLOG(error, #{ - msg => "action_references_nonexistent_bridges", + ?tp(error, "action_references_nonexistent_bridges", #{ rule_id => RuleId, nonexistent_bridge_ids => NonExistentBridgeIDs, hint => "this rule will be disabled" @@ -626,7 +626,7 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeID, #{atom_name => false}), case emqx_action_info:is_action_type(Type) of - true -> {action, Type, Name}; + true -> {source, Type, Name}; false -> {bridge_v1, Type, Name} end end, @@ -646,7 +646,8 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul fun({Kind, Type, Name}) -> LookupFn = case Kind of - action -> fun emqx_bridge_v2:lookup/2; + action -> fun emqx_bridge_v2:lookup_action/2; + source -> fun emqx_bridge_v2:lookup_source/2; bridge_v1 -> fun emqx_bridge:lookup/2 end, try From 8f304d3456688cd2b7d73703898b859ea178d42b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 16:24:52 -0300 Subject: [PATCH 20/28] test(bridge_v2_api): refactor suite to use CT matrix --- apps/emqx/test/emqx_common_test_helpers.erl | 29 +- .../test/emqx_bridge_v2_api_SUITE.erl | 1230 +++++++++-------- 2 files changed, 669 insertions(+), 590 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d9c9470eb..9438d227e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1389,29 +1389,40 @@ matrix_to_groups(Module, Cases) -> Cases ). -add_case_matrix(Module, Case, Acc0) -> - {RootGroup, Matrix} = Module:Case(matrix), +add_case_matrix(Module, TestCase, Acc0) -> + {MaybeRootGroup, Matrix} = + case Module:TestCase(matrix) of + {RootGroup0, Matrix0} -> + {RootGroup0, Matrix0}; + Matrix0 -> + {undefined, Matrix0} + end, lists:foldr( fun(Row, Acc) -> - add_group([RootGroup | Row], Acc, Case) + case MaybeRootGroup of + undefined -> + add_group(Row, Acc, TestCase); + RootGroup -> + add_group([RootGroup | Row], Acc, TestCase) + end end, Acc0, Matrix ). -add_group([], Acc, Case) -> - case lists:member(Case, Acc) of +add_group([], Acc, TestCase) -> + case lists:member(TestCase, Acc) of true -> Acc; false -> - [Case | Acc] + [TestCase | Acc] end; -add_group([Name | More], Acc, Cases) -> +add_group([Name | More], Acc, TestCases) -> case lists:keyfind(Name, 1, Acc) of false -> - [{Name, [], add_group(More, [], Cases)} | Acc]; + [{Name, [], add_group(More, [], TestCases)} | Acc]; {Name, [], SubGroup} -> - New = {Name, [], add_group(More, SubGroup, Cases)}, + New = {Name, [], add_group(More, SubGroup, TestCases)}, lists:keystore(Name, 1, Acc, New) end. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 0c34610ea..0ce8c620e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -24,9 +24,9 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). --define(ROOT, "actions"). +-define(ACTIONS_ROOT, "actions"). --define(CONNECTOR_NAME, <<"my_connector">>). +-define(ACTION_CONNECTOR_NAME, <<"my_connector">>). -define(RESOURCE(NAME, TYPE), #{ <<"enable">> => true, @@ -35,10 +35,10 @@ <<"name">> => NAME }). --define(CONNECTOR_TYPE_STR, "kafka_producer"). --define(CONNECTOR_TYPE, <>). +-define(ACTION_CONNECTOR_TYPE_STR, "kafka_producer"). +-define(ACTION_CONNECTOR_TYPE, <>). -define(KAFKA_BOOTSTRAP_HOST, <<"127.0.0.1:9092">>). --define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?CONNECTOR_TYPE)#{ +-define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?ACTION_CONNECTOR_TYPE)#{ <<"authentication">> => <<"none">>, <<"bootstrap_hosts">> => BootstrapHosts, <<"connect_timeout">> => <<"5s">>, @@ -53,14 +53,14 @@ } }). --define(CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). --define(CONNECTOR, ?CONNECTOR(?CONNECTOR_NAME)). +-define(ACTIONS_CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). +-define(ACTIONS_CONNECTOR, ?ACTIONS_CONNECTOR(?ACTION_CONNECTOR_NAME)). -define(MQTT_LOCAL_TOPIC, <<"mqtt/local/topic">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). --define(BRIDGE_TYPE_STR, "kafka_producer"). --define(BRIDGE_TYPE, <>). --define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?BRIDGE_TYPE)#{ +-define(ACTION_TYPE_STR, "kafka_producer"). +-define(ACTION_TYPE, <>). +-define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?ACTION_TYPE)#{ <<"connector">> => Connector, <<"kafka">> => #{ <<"buffer">> => #{ @@ -99,12 +99,12 @@ <<"health_check_interval">> => <<"32s">> } }). --define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?ACTION_CONNECTOR_NAME)). -define(KAFKA_BRIDGE_UPDATE(Name, Connector), maps:without([<<"name">>, <<"type">>], ?KAFKA_BRIDGE(Name, Connector)) ). --define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?ACTION_CONNECTOR_NAME)). -define(APPSPECS, [ emqx_conf, @@ -120,34 +120,27 @@ {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ). +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + -if(?EMQX_RELEASE_EDITION == ee). %% For now we got only kafka implementing `bridge_v2` and that is enterprise only. all() -> - [ - {group, single}, - {group, cluster_later_join}, - {group, cluster} - ]. + All0 = emqx_common_test_helpers:all(?MODULE), + All = All0 -- matrix_cases(), + Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()), + Groups ++ All. -else. all() -> []. -endif. +matrix_cases() -> + emqx_common_test_helpers:all(?MODULE). + groups() -> - AllTCs = emqx_common_test_helpers:all(?MODULE), - SingleOnlyTests = [ - t_bridges_probe, - t_broken_bridge_config, - t_fix_broken_bridge_config - ], - ClusterLaterJoinOnlyTCs = [ - t_cluster_later_join_metrics - ], - [ - {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, - {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, - {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} - ]. + emqx_common_test_helpers:matrix_to_groups(?MODULE, matrix_cases()). suite() -> [{timetrap, {seconds, 60}}]. @@ -164,10 +157,12 @@ init_per_group(cluster = Name, Config) -> init_per_group(cluster_later_join = Name, Config) -> Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); -init_per_group(Name, Config) -> - WorkDir = filename:join(?config(priv_dir, Config), Name), +init_per_group(single = Group, Config) -> + WorkDir = filename:join(?config(priv_dir, Config), Group), Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), - init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]). + init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]); +init_per_group(actions, Config) -> + [{bridge_kind, action} | Config]. init_api(Config) -> Node = ?config(node, Config), @@ -193,8 +188,10 @@ end_per_group(Group, Config) when Group =:= cluster_later_join -> ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)); -end_per_group(_, Config) -> +end_per_group(single, Config) -> emqx_cth_suite:stop(?config(group_apps, Config)), + ok; +end_per_group(_Group, _Config) -> ok. init_per_testcase(t_action_types, Config) -> @@ -212,7 +209,7 @@ init_per_testcase(_TestCase, Config) -> Nodes -> [erpc:call(Node, ?MODULE, init_mocks, []) || Node <- Nodes] end, - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR, Config), + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config), Config. end_per_testcase(_TestCase, Config) -> @@ -227,6 +224,10 @@ end_per_testcase(_TestCase, Config) -> ok = emqx_common_test_helpers:call_janitor(), ok. +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + -define(CONNECTOR_IMPL, emqx_bridge_v2_dummy_connector). init_mocks() -> case emqx_release:edition() of @@ -243,7 +244,7 @@ init_mocks() -> ?CONNECTOR_IMPL, on_start, fun - (<<"connector:", ?CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> + (<<"connector:", ?ACTION_CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> {ok, bad_connector_state}; (_I, _C) -> {ok, connector_state} @@ -280,442 +281,6 @@ clear_resources() -> emqx_connector:list() ). -%%------------------------------------------------------------------------------ -%% Testcases -%%------------------------------------------------------------------------------ - -%% We have to pretend testing a kafka bridge since at this point that's the -%% only one that's implemented. - -t_bridges_lifecycle(Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - {ok, 404, _} = request(get, uri([?ROOT, "foo"]), Config), - {ok, 404, _} = request(get, uri([?ROOT, "kafka_producer:foo"]), Config), - - %% need a var for patterns below - BridgeName = ?BRIDGE_NAME, - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"connector">> := ?CONNECTOR_NAME, - <<"parameters">> := #{}, - <<"local_topic">> := _, - <<"resource_opts">> := _ - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME), - Config - ) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% get the bridge by id - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ?assertMatch( - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := _ - }}, - request_json(post, uri([?ROOT, BridgeID, "brababbel"]), Config) - ), - - %% update bridge config - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR(<<"foobla">>), Config), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"connector">> := <<"foobla">>, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), - Config - ) - ), - - %% update bridge with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message1 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message1) - ), - - %% update bridge with connector of wrong type - {ok, 201, _} = - request( - post, - uri(["connectors"]), - (?CONNECTOR(<<"foobla2">>))#{ - <<"type">> => <<"azure_event_hub_producer">>, - <<"authentication">> => #{ - <<"username">> => <<"emqxuser">>, - <<"password">> => <<"topSecret">>, - <<"mechanism">> => <<"plain">> - }, - <<"ssl">> => #{ - <<"enable">> => true, - <<"server_name_indication">> => <<"auto">>, - <<"verify">> => <<"verify_none">>, - <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] - } - }, - Config - ), - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message2 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message2) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% try create with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message3 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message3) - ), - - %% try create bridge with connector of wrong type - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message4 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message4) - ), - - %% make sure nothing has been created above - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% update a deleted bridge returns an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), - Config - ) - ), - - %% deleting a non-existing bridge should result in an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json(delete, uri([?ROOT, BridgeID]), Config) - ), - - %% try delete unknown bridge id - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := <<"Invalid bridge ID", _/binary>> - }}, - request_json(delete, uri([?ROOT, "foo"]), Config) - ), - - %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), - ok. - -t_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertEqual( - {ok, 204, <<>>}, - request(delete, uri([?ROOT, BridgeID]), Config) - ), - - ?assertEqual( - {ok, 200, []}, - request_json(get, uri([?ROOT]), Config) - ), - - ok. - -t_fix_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?CONNECTOR_NAME), - Config - ), - - ?assertMatch( - {ok, 200, #{ - <<"connector">> := ?CONNECTOR_NAME, - <<"status">> := <<"connected">> - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ok. - -t_start_bridge_unknown_node(Config) -> - {ok, 404, _} = - request( - post, - uri(["nodes", "thisbetterbenotanatomyet", ?ROOT, "kafka_producer:foo", start]), - Config - ), - {ok, 404, _} = - request( - post, - uri(["nodes", "undefined", ?ROOT, "kafka_producer:foo", start]), - Config - ). - -t_start_bridge_node(Config) -> - do_start_bridge(node, Config). - -t_start_bridge_cluster(Config) -> - do_start_bridge(cluster, Config). - -do_start_bridge(TestType, Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - Name = atom_to_binary(TestType), - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _] - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(Name), - Config - ) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), - - %% start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - %% start a started bridge - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), - - %% Make start bridge fail - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, - Config - ), - - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, stop), - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, start), - - {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% Make start bridge succeed - - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, - Config - ), - - %% try to start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% Fail parse-id check - {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), - %% Looks ok but doesn't exist - {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), - ok. - expect_on_all_nodes(Mod, Function, Fun, Config) -> case ?config(cluster_nodes, Config) of undefined -> @@ -751,6 +316,548 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> ok = emqx_connector_resource:OperationName(ConnectorType, ConnectorName) end. +listen_on_random_port() -> + SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], + case gen_tcp:listen(0, SockOpts) of + {ok, Sock} -> + {ok, Port} = inet:port(Sock), + {Port, Sock}; + {error, Reason} when Reason /= eaddrinuse -> + {error, Reason} + end. + +request(Method, URL, Config) -> + request(Method, URL, [], Config). + +request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> + URL = operation_path(Type, Op, BridgeID, Config), + request(Method, URL, Body, Config); +request(Method, URL, Body, Config) -> + AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). + +request(Method, URL, Body, Decoder, Config) -> + case request(Method, URL, Body, Config) of + {ok, Code, Response} -> + case Decoder(Response) of + {error, _} = Error -> Error; + Decoded -> {ok, Code, Decoded} + end; + Otherwise -> + Otherwise + end. + +request_json(Method, URLLike, Config) -> + request(Method, URLLike, [], fun json/1, Config). + +request_json(Method, URLLike, Body, Config) -> + request(Method, URLLike, Body, fun json/1, Config). + +operation_path(node, Oper, BridgeID, Config) -> + uri(["nodes", ?config(node, Config), ?ACTIONS_ROOT, BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, _Config) -> + uri([?ACTIONS_ROOT, BridgeID, Oper]). + +enable_path(Enable, BridgeID) -> + uri([?ACTIONS_ROOT, BridgeID, "enable", Enable]). + +publish_message(Topic, Body, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). + +update_config(Path, Value, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, update_config, [Path, Value]). + +get_raw_config(Path, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, get_raw_config, [Path]). + +add_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). + +delete_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). + +str(S) when is_list(S) -> S; +str(S) when is_binary(S) -> binary_to_list(S). + +json(B) when is_binary(B) -> + case emqx_utils_json:safe_decode(B, [return_maps]) of + {ok, Term} -> + Term; + {error, Reason} = Error -> + ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), + Error + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +%% We have to pretend testing a kafka bridge since at this point that's the +%% only one that's implemented. + +t_bridges_lifecycle(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_bridges_lifecycle(Config) -> + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "foo"]), Config), + {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "kafka_producer:foo"]), Config), + + %% need a var for patterns below + BridgeName = ?BRIDGE_NAME, + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"parameters">> := #{}, + <<"local_topic">> := _, + <<"resource_opts">> := _ + }}, + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME), + Config + ) + ), + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + %% get the bridge by id + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + ?assertMatch( + {ok, 200, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := _ + }}, + request_json(post, uri([?ACTIONS_ROOT, BridgeID, "brababbel"]), Config) + ), + + %% update bridge config + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR(<<"foobla">>), Config), + ?assertMatch( + {ok, 200, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"connector">> := <<"foobla">>, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), + Config + ) + ), + + %% update bridge with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message1 + }} = + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message1) + ), + + %% update bridge with connector of wrong type + {ok, 201, _} = + request( + post, + uri(["connectors"]), + (?ACTIONS_CONNECTOR(<<"foobla2">>))#{ + <<"type">> => <<"azure_event_hub_producer">>, + <<"authentication">> => #{ + <<"username">> => <<"emqxuser">>, + <<"password">> => <<"topSecret">>, + <<"mechanism">> => <<"plain">> + }, + <<"ssl">> => #{ + <<"enable">> => true, + <<"server_name_indication">> => <<"auto">>, + <<"verify">> => <<"verify_none">>, + <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] + } + }, + Config + ), + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message2 + }} = + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message2) + ), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% try create with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message3 + }} = + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message3) + ), + + %% try create bridge with connector of wrong type + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message4 + }} = + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message4) + ), + + %% make sure nothing has been created above + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% update a deleted bridge returns an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), + Config + ) + ), + + %% deleting a non-existing bridge should result in an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + %% try delete unknown bridge id + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Invalid bridge ID", _/binary>> + }}, + request_json(delete, uri([?ACTIONS_ROOT, "foo"]), Config) + ), + + %% Try create bridge with bad characters as name + {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), + {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), + ok. + +t_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + ?assertEqual( + {ok, 204, <<>>}, + request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ?assertEqual( + {ok, 200, []}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + ok. + +t_fix_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_fix_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?ACTION_CONNECTOR_NAME), + Config + ), + + ?assertMatch( + {ok, 200, #{ + <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"status">> := <<"connected">> + }}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ok. + +t_start_bridge_unknown_node(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_unknown_node(Config) -> + {ok, 404, _} = + request( + post, + uri(["nodes", "thisbetterbenotanatomyet", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ), + {ok, 404, _} = + request( + post, + uri(["nodes", "undefined", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ). + +t_start_bridge_node(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_node(Config) -> + do_start_bridge(node, Config). + +t_start_bridge_cluster(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_cluster(Config) -> + do_start_bridge(cluster, Config). + +do_start_bridge(TestType, Config) -> + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + Name = atom_to_binary(TestType), + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _] + }}, + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(Name), + Config + ) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), + + %% start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + %% start a started bridge + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), + + %% Make start bridge fail + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, + Config + ), + + connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, stop), + connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, start), + + {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% Make start bridge succeed + + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, + Config + ), + + %% try to start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% Fail parse-id check + {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), + %% Looks ok but doesn't exist + {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), + ok. + %% t_start_stop_inconsistent_bridge_node(Config) -> %% start_stop_inconsistent_bridge(node, Config). @@ -861,6 +968,10 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> %% {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), %% {ok, 200, []} = request_json(get, uri([?ROOT]), Config). +t_bridges_probe(matrix) -> + [ + [single, actions] + ]; t_bridges_probe(Config) -> {ok, 204, <<>>} = request( post, @@ -905,15 +1016,20 @@ t_bridges_probe(Config) -> ), ok. +t_cascade_delete_actions(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_cascade_delete_actions(Config) -> %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% then we add a a bridge, using POST %% POST /actions/ will create a bridge - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -931,10 +1047,10 @@ t_cascade_delete_actions(Config) -> %% delete the bridge will also delete the actions from the rules {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertMatch( {ok, 200, #{<<"actions">> := []}}, request_json(get, uri(["rules", RuleId]), Config) @@ -943,7 +1059,7 @@ t_cascade_delete_actions(Config) -> {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -960,19 +1076,24 @@ t_cascade_delete_actions(Config) -> ), {ok, 400, Body} = request( delete, - uri([?ROOT, BridgeID]), + uri([?ACTIONS_ROOT, BridgeID]), Config ), ?assertMatch(#{<<"rules">> := [_ | _]}, emqx_utils_json:decode(Body, [return_maps])), - {ok, 200, [_]} = request_json(get, uri([?ROOT]), Config), + {ok, 200, [_]} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% Cleanup {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config). + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config). +t_action_types(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_action_types(Config) -> Res = request_json(get, uri(["action_types"]), Config), ?assertMatch({ok, 200, _}, Res), @@ -981,11 +1102,16 @@ t_action_types(Config) -> ?assert(lists:all(fun is_binary/1, Types), #{types => Types}), ok. +t_bad_name(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_bad_name(Config) -> Name = <<"_bad_name">>, Res = request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(Name), Config ), @@ -1001,31 +1127,36 @@ t_bad_name(Config) -> ), ok. +t_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_metrics(Config) -> - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 0}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), - {ok, 200, Bridge} = request_json(get, uri([?ROOT, ActionID]), Config), + {ok, 200, Bridge} = request_json(get, uri([?ACTIONS_ROOT, ActionID]), Config), ?assertNot(maps:is_key(<<"metrics">>, Bridge)), ?assertNot(maps:is_key(<<"node_metrics">>, Bridge)), @@ -1041,12 +1172,12 @@ t_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 1}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), %% check for absence of metrics when listing all bridges - {ok, 200, Bridges} = request_json(get, uri([?ROOT]), Config), + {ok, 200, Bridges} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertNotMatch( [ #{ @@ -1058,21 +1189,26 @@ t_metrics(Config) -> ), ok. +t_reset_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_reset_metrics(Config) -> %% assert there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), Body = <<"my msg">>, _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), @@ -1084,11 +1220,11 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), - {ok, 204, <<>>} = request(put, uri([?ROOT, ActionID, "metrics", "reset"]), Config), + {ok, 204, <<>>} = request(put, uri([?ACTIONS_ROOT, ActionID, "metrics", "reset"]), Config), ?retry( _Sleep0 = 200, @@ -1098,28 +1234,34 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), ok. +t_cluster_later_join_metrics(matrix) -> + [ + [cluster_later_join, actions] + ]; t_cluster_later_join_metrics(Config) -> [PrimaryNode, OtherNode | _] = ?config(cluster_nodes, Config), Name = ?BRIDGE_NAME, ActionParams = ?KAFKA_BRIDGE(Name), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), ?check_trace( begin %% Create a bridge on only one of the nodes. - ?assertMatch({ok, 201, _}, request_json(post, uri([?ROOT]), ActionParams, Config)), + ?assertMatch( + {ok, 201, _}, request_json(post, uri([?ACTIONS_ROOT]), ActionParams, Config) + ), %% Pre-condition. ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), %% Now join the other node join with the api node. ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]), @@ -1130,7 +1272,7 @@ t_cluster_later_join_metrics(Config) -> <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), ok end, @@ -1138,94 +1280,20 @@ t_cluster_later_join_metrics(Config) -> ), ok. +t_raw_config_response_defaults(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_raw_config_response_defaults(Config) -> Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), ?assertMatch( {ok, 201, #{<<"enable">> := true}}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), Params, Config ) ), ok. - -%%% helpers -listen_on_random_port() -> - SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], - case gen_tcp:listen(0, SockOpts) of - {ok, Sock} -> - {ok, Port} = inet:port(Sock), - {Port, Sock}; - {error, Reason} when Reason /= eaddrinuse -> - {error, Reason} - end. - -request(Method, URL, Config) -> - request(Method, URL, [], Config). - -request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> - URL = operation_path(Type, Op, BridgeID, Config), - request(Method, URL, Body, Config); -request(Method, URL, Body, Config) -> - AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), - Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, - emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). - -request(Method, URL, Body, Decoder, Config) -> - case request(Method, URL, Body, Config) of - {ok, Code, Response} -> - case Decoder(Response) of - {error, _} = Error -> Error; - Decoded -> {ok, Code, Decoded} - end; - Otherwise -> - Otherwise - end. - -request_json(Method, URLLike, Config) -> - request(Method, URLLike, [], fun json/1, Config). - -request_json(Method, URLLike, Body, Config) -> - request(Method, URLLike, Body, fun json/1, Config). - -operation_path(node, Oper, BridgeID, Config) -> - uri(["nodes", ?config(node, Config), ?ROOT, BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID, _Config) -> - uri([?ROOT, BridgeID, Oper]). - -enable_path(Enable, BridgeID) -> - uri([?ROOT, BridgeID, "enable", Enable]). - -publish_message(Topic, Body, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). - -update_config(Path, Value, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, update_config, [Path, Value]). - -get_raw_config(Path, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, get_raw_config, [Path]). - -add_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). - -delete_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). - -str(S) when is_list(S) -> S; -str(S) when is_binary(S) -> binary_to_list(S). - -json(B) when is_binary(B) -> - case emqx_utils_json:safe_decode(B, [return_maps]) of - {ok, Term} -> - Term; - {error, Reason} = Error -> - ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), - Error - end. From fc88a1ed1ee2b4818a0ca725c03aae3b177c1538 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 17:27:23 -0300 Subject: [PATCH 21/28] test(sources_api): add some tests to cover `/sources` HTTP API Also fixes a bug with `DELETE /sources/:id` --- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 2 +- .../test/emqx_bridge_v2_api_SUITE.erl | 277 ++++++++++++++---- 2 files changed, 221 insertions(+), 58 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index d4401cfd0..e8a500e85 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -786,7 +786,7 @@ handle_update(ConfRootKey, Id, Conf0) -> handle_delete(ConfRootKey, Id, QueryStringOpts) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> AlsoDeleteActions = case maps:get(<<"also_delete_dep_actions">>, QueryStringOpts, <<"false">>) of diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 0ce8c620e..5ef897369 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -25,8 +25,10 @@ -include_lib("snabbkaffe/include/test_macros.hrl"). -define(ACTIONS_ROOT, "actions"). +-define(SOURCES_ROOT, "sources"). -define(ACTION_CONNECTOR_NAME, <<"my_connector">>). +-define(SOURCE_CONNECTOR_NAME, <<"my_connector">>). -define(RESOURCE(NAME, TYPE), #{ <<"enable">> => true, @@ -106,6 +108,9 @@ ). -define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?ACTION_CONNECTOR_NAME)). +-define(SOURCE_TYPE_STR, "mqtt"). +-define(SOURCE_TYPE, <>). + -define(APPSPECS, [ emqx_conf, emqx, @@ -162,7 +167,11 @@ init_per_group(single = Group, Config) -> Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]); init_per_group(actions, Config) -> - [{bridge_kind, action} | Config]. + [{bridge_kind, action} | Config]; +init_per_group(sources, Config) -> + [{bridge_kind, source} | Config]; +init_per_group(_Group, Config) -> + Config. init_api(Config) -> Node = ?config(node, Config), @@ -209,7 +218,17 @@ init_per_testcase(_TestCase, Config) -> Nodes -> [erpc:call(Node, ?MODULE, init_mocks, []) || Node <- Nodes] end, - {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config), + case ?config(bridge_kind, Config) of + action -> + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config); + source -> + {ok, 201, _} = request( + post, + uri(["connectors"]), + source_connector_create_config(#{}), + Config + ) + end, Config. end_per_testcase(_TestCase, Config) -> @@ -268,18 +287,7 @@ init_mocks() -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ). + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(). expect_on_all_nodes(Mod, Function, Fun, Config) -> case ?config(cluster_nodes, Config) of @@ -394,6 +402,135 @@ json(B) when is_binary(B) -> Error end. +group_path(Config) -> + case emqx_common_test_helpers:group_path(Config) of + [] -> + undefined; + Path -> + Path + end. + +source_connector_config_base() -> + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_connector_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_connector_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE, + <<"name">> => ?SOURCE_CONNECTOR_NAME + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_config_base() -> + #{ + <<"enable">> => true, + <<"connector">> => ?SOURCE_CONNECTOR_NAME, + <<"parameters">> => + #{ + <<"remote">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + } + }, + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"batch_time">> => <<"0ms">>, + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"15s">>, + <<"inflight_window">> => 100, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"45s">>, + <<"resume_interval">> => <<"15s">>, + <<"worker_pool_size">> => <<"1">> + } + }. + +source_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_update_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + maps:merge( + source_config_base(), + Overrides + ). + +get_common_values(Kind, FnName) -> + case Kind of + actions -> + #{ + api_root_key => ?ACTIONS_ROOT, + type => ?ACTION_TYPE, + default_connector_name => ?ACTION_CONNECTOR_NAME, + create_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE(Name, ConnectorName) + end, + update_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE_UPDATE(Name, ConnectorName) + end, + create_connector_config_fn => + fun(Overrides) -> + ConnectorName = maps:get(name, Overrides, ?ACTION_CONNECTOR_NAME), + ?ACTIONS_CONNECTOR(ConnectorName) + end + }; + sources -> + #{ + api_root_key => ?SOURCES_ROOT, + type => ?SOURCE_TYPE, + default_connector_name => ?SOURCE_CONNECTOR_NAME, + create_config_fn => fun(Overrides0) -> + Overrides = + case Overrides0 of + #{name := _} -> Overrides0; + _ -> Overrides0#{name => FnName} + end, + source_create_config(Overrides) + end, + update_config_fn => fun source_update_config/1, + create_connector_config_fn => fun source_connector_create_config/1 + } + end. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -404,76 +541,95 @@ json(B) when is_binary(B) -> t_bridges_lifecycle(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_bridges_lifecycle(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + FnName = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn, + update_config_fn := UpdateConfigFn, + create_connector_config_fn := CreateConnectorConfigFn + } = get_common_values(Kind, FnName), %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), - {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "foo"]), Config), - {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "kafka_producer:foo"]), Config), + {ok, 404, _} = request(get, uri([APIRootKey, "foo"]), Config), + {ok, 404, _} = request(get, uri([APIRootKey, "kafka_producer:foo"]), Config), %% need a var for patterns below - BridgeName = ?BRIDGE_NAME, + BridgeName = FnName, + CreateRes = request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{}), + Config + ), ?assertMatch( {ok, 201, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], - <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"connector">> := DefaultConnectorName, <<"parameters">> := #{}, - <<"local_topic">> := _, <<"resource_opts">> := _ }}, - request_json( - post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME), - Config - ) + CreateRes, + #{name => BridgeName, type => Type, connector => DefaultConnectorName} ), + case Kind of + actions -> + ?assertMatch({ok, 201, #{<<"local_topic">> := _}}, CreateRes); + sources -> + ok + end, %% list all bridges, assert bridge is in it ?assertMatch( {ok, 200, [ #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] } ]}, - request_json(get, uri([?ACTIONS_ROOT]), Config) + request_json(get, uri([APIRootKey]), Config) ), %% list all bridges, assert bridge is in it ?assertMatch( {ok, 200, [ #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] } ]}, - request_json(get, uri([?ACTIONS_ROOT]), Config) + request_json(get, uri([APIRootKey]), Config) ), %% get the bridge by id - BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(Type, ?BRIDGE_NAME), ?assertMatch( {ok, 200, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] }}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), ?assertMatch( @@ -481,14 +637,19 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := _ }}, - request_json(post, uri([?ACTIONS_ROOT, BridgeID, "brababbel"]), Config) + request_json(post, uri([APIRootKey, BridgeID, "brababbel"]), Config) ), %% update bridge config - {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR(<<"foobla">>), Config), + {ok, 201, _} = request( + post, + uri(["connectors"]), + CreateConnectorConfigFn(#{name => <<"foobla">>}), + Config + ), ?assertMatch( {ok, 200, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"connector">> := <<"foobla">>, <<"enable">> := true, @@ -497,8 +658,8 @@ t_bridges_lifecycle(Config) -> }}, request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla">>}), Config ) ), @@ -510,8 +671,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"does_not_exist">>}), Config ), ?assertMatch( @@ -546,8 +707,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla2">>}), Config ), ?assertMatch( @@ -556,8 +717,8 @@ t_bridges_lifecycle(Config) -> ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% try create with unknown connector name {ok, 400, #{ @@ -566,8 +727,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"does_not_exist">>}), Config ), ?assertMatch( @@ -582,8 +743,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"foobla2">>}), Config ), ?assertMatch( @@ -592,7 +753,7 @@ t_bridges_lifecycle(Config) -> ), %% make sure nothing has been created above - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% update a deleted bridge returns an error ?assertMatch( @@ -602,8 +763,8 @@ t_bridges_lifecycle(Config) -> }}, request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{}), Config ) ), @@ -614,7 +775,7 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"NOT_FOUND">>, <<"message">> := _ }}, - request_json(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(delete, uri([APIRootKey, BridgeID]), Config) ), %% try delete unknown bridge id @@ -623,12 +784,14 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Invalid bridge ID", _/binary>> }}, - request_json(delete, uri([?ACTIONS_ROOT, "foo"]), Config) + request_json(delete, uri([APIRootKey, "foo"]), Config) ), %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), - {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), + {ok, 400, _} = request( + post, uri([APIRootKey]), CreateConfigFn(#{name => <<"隋达"/utf8>>}), Config + ), + {ok, 400, _} = request(post, uri([APIRootKey]), CreateConfigFn(#{name => <<"a.b">>}), Config), ok. t_broken_bridge_config(matrix) -> From 007af20a30472a1b22c5615c50c6f6d2b1a28d76 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 17:59:11 -0300 Subject: [PATCH 22/28] test(bridge_v2_api): adapt more tests to sources --- .../test/emqx_bridge_v2_api_SUITE.erl | 80 +++++++++++++------ 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 5ef897369..d24d4feac 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -255,6 +255,8 @@ init_mocks() -> meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), ok; ce -> + meck:new(emqx_connector_resource, [passthrough, no_link]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), ok end, meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), @@ -363,9 +365,13 @@ request_json(Method, URLLike, Body, Config) -> request(Method, URLLike, Body, fun json/1, Config). operation_path(node, Oper, BridgeID, Config) -> - uri(["nodes", ?config(node, Config), ?ACTIONS_ROOT, BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID, _Config) -> - uri([?ACTIONS_ROOT, BridgeID, Oper]). + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri(["nodes", ?config(node, Config), APIRootKey, BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri([APIRootKey, BridgeID, Oper]). enable_path(Enable, BridgeID) -> uri([?ACTIONS_ROOT, BridgeID, "enable", Enable]). @@ -935,7 +941,9 @@ t_start_bridge_unknown_node(Config) -> t_start_bridge_node(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_start_bridge_node(Config) -> do_start_bridge(node, Config). @@ -943,19 +951,28 @@ t_start_bridge_node(Config) -> t_start_bridge_cluster(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_start_bridge_cluster(Config) -> do_start_bridge(cluster, Config). do_start_bridge(TestType, Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), - + [_SingleOrCluster, Kind | _] = group_path(Config), Name = atom_to_binary(TestType), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + ?assertMatch( {ok, 201, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, @@ -963,25 +980,25 @@ do_start_bridge(TestType, Config) -> }}, request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(Name), + uri([APIRootKey]), + CreateConfigFn(#{name => Name}), Config ) ), - BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(Type, Name), %% start again {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), ?assertMatch( {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), %% start a started bridge {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), ?assertMatch( {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), @@ -994,8 +1011,8 @@ do_start_bridge(TestType, Config) -> Config ), - connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, stop), - connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, start), + connector_operation(Config, Type, DefaultConnectorName, stop), + connector_operation(Config, Type, DefaultConnectorName, start), {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), @@ -1012,8 +1029,8 @@ do_start_bridge(TestType, Config) -> {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% Fail parse-id check {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), @@ -1268,14 +1285,21 @@ t_action_types(Config) -> t_bad_name(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_bad_name(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), Name = <<"_bad_name">>, + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), Res = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(Name), + uri([APIRootKey]), + CreateConfigFn(#{}), Config ), ?assertMatch({ok, 400, #{<<"message">> := _}}, Res), @@ -1446,15 +1470,23 @@ t_cluster_later_join_metrics(Config) -> t_raw_config_response_defaults(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_raw_config_response_defaults(Config) -> - Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), + [_SingleOrCluster, Kind | _] = group_path(Config), + Name = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + Params = maps:remove(<<"enable">>, CreateConfigFn(#{})), ?assertMatch( {ok, 201, #{<<"enable">> := true}}, request_json( post, - uri([?ACTIONS_ROOT]), + uri([APIRootKey]), Params, Config ) From 938429f3518359ac1ef148ca38a3d728bed8ddf0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jan 2024 13:55:11 -0300 Subject: [PATCH 23/28] chore(mqtt_bridge): change schema to remote `remote` sub-fields and hide `local` `local` is still needed for backwards compatibility --- .../test/emqx_bridge_v2_api_SUITE.erl | 7 +- .../src/emqx_bridge_mqtt_connector.erl | 28 +++++--- .../src/emqx_bridge_mqtt_ingress.erl | 10 +-- .../emqx_bridge_mqtt_pubsub_action_info.erl | 65 +++++++++++++------ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 36 +++++++--- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 9 +-- 6 files changed, 101 insertions(+), 54 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index d24d4feac..fabaadb92 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -451,11 +451,8 @@ source_config_base() -> <<"connector">> => ?SOURCE_CONNECTOR_NAME, <<"parameters">> => #{ - <<"remote">> => - #{ - <<"topic">> => <<"remote/topic">>, - <<"qos">> => 2 - } + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 }, <<"resource_opts">> => #{ <<"batch_size">> => 1, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 18af6ee11..9aae73bd2 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -127,8 +127,9 @@ on_add_channel( true -> ok end, - ChannelState0 = maps:get(parameters, ChannelConfig), - ChannelState = emqx_bridge_mqtt_egress:config(ChannelState0), + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState = emqx_bridge_mqtt_egress:config(#{remote => RemoteParams, local => LocalParams}), NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels), NewState = OldState#{installed_channels => NewInstalledChannels}, {ok, NewState}; @@ -144,15 +145,18 @@ on_add_channel( #{hookpoints := HookPoints} = ChannelConfig ) -> %% Add ingress channel - ChannelState0 = maps:get(parameters, ChannelConfig), - ChannelState1 = ChannelState0#{ + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState0 = #{ hookpoints => HookPoints, server => Server, - config_root => sources + config_root => sources, + local => LocalParams, + remote => RemoteParams }, - ChannelState2 = mk_ingress_config(ChannelId, ChannelState1, TopicToHandlerIndex), - ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState2), - NewInstalledChannels = maps:put(ChannelId, ChannelState2, InstalledChannels), + ChannelState1 = mk_ingress_config(ChannelId, ChannelState0, TopicToHandlerIndex), + ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState1), + NewInstalledChannels = maps:put(ChannelId, ChannelState1, InstalledChannels), NewState = OldState#{installed_channels => NewInstalledChannels}, {ok, NewState}. @@ -500,3 +504,11 @@ connect(Pid, Name) -> handle_disconnect(_Reason) -> ok. + +take(Key, Map0, Default) -> + case maps:take(Key, Map0) of + {Value, Map} -> + {Value, Map}; + error -> + {Default, Map0} + end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index d59318a84..369238ecf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -178,11 +178,13 @@ config(#{ingress_list := IngressList} = Conf, Name, TopicToHandlerIndex) -> ], Conf#{ingress_list => NewIngressList}. -fix_remote_config(#{remote := RC, local := LC}, BridgeName, TopicToHandlerIndex, Conf) -> - FixedConf = Conf#{ - remote => parse_remote(RC, BridgeName), - local => emqx_bridge_mqtt_msg:parse(LC) +fix_remote_config(#{remote := RC}, BridgeName, TopicToHandlerIndex, Conf) -> + FixedConf0 = Conf#{ + remote => parse_remote(RC, BridgeName) }, + FixedConf = emqx_utils_maps:update_if_present( + local, fun emqx_bridge_mqtt_msg:parse/1, FixedConf0 + ), insert_to_topic_to_handler_index(FixedConf, TopicToHandlerIndex, BridgeName), FixedConf. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index e4a4fcd19..cf7a5bc04 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -95,8 +95,11 @@ bridge_v1_config_to_action_config_helper( LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), EgressMap1 = maps:without([<<"local">>, <<"pool_size">>], EgressMap0), + LocalParams = maps:get(<<"local">>, EgressMap0, #{}), + EgressMap2 = emqx_utils_maps:unindent(<<"remote">>, EgressMap1), + EgressMap = maps:put(<<"local">>, LocalParams, EgressMap2), %% Add parameters field (Egress map) to the action config - ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), + ConfigMap2 = maps:put(<<"parameters">>, EgressMap, ConfigMap1), ConfigMap3 = case LocalTopic of undefined -> @@ -107,7 +110,7 @@ bridge_v1_config_to_action_config_helper( {action, mqtt, ConfigMap3}; bridge_v1_config_to_action_config_helper( #{ - <<"ingress">> := IngressMap + <<"ingress">> := IngressMap0 } = Config, ConnectorName ) -> @@ -117,9 +120,12 @@ bridge_v1_config_to_action_config_helper( ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), - IngressMap1 = maps:remove(<<"pool_size">>, IngressMap), + IngressMap1 = maps:without([<<"pool_size">>, <<"local">>], IngressMap0), + LocalParams = maps:get(<<"local">>, IngressMap0, #{}), + IngressMap2 = emqx_utils_maps:unindent(<<"remote">>, IngressMap1), + IngressMap = maps:put(<<"local">>, LocalParams, IngressMap2), %% Add parameters field (Egress map) to the action config - ConfigMap2 = maps:put(<<"parameters">>, IngressMap1, ConfigMap1), + ConfigMap2 = maps:put(<<"parameters">>, IngressMap, ConfigMap1), {source, mqtt, ConfigMap2}; bridge_v1_config_to_action_config_helper( _Config, @@ -182,7 +188,7 @@ check_and_simplify_bridge_v1_config(SimplifiedConfig) -> connector_action_config_to_bridge_v1_config( ConnectorConfig, ActionConfig ) -> - Params = maps:get(<<"parameters">>, ActionConfig, #{}), + Params0 = maps:get(<<"parameters">>, ActionConfig, #{}), ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), ResourceOpts0 = maps:merge(ResourceOptsConnector, ResourceOptsAction), @@ -194,37 +200,54 @@ connector_action_config_to_bridge_v1_config( ResourceOpts = maps:with(V1ResourceOptsFields, ResourceOpts0), %% Check the direction of the action Direction = - case maps:get(<<"remote">>, Params) of - #{<<"retain">> := _} -> - %% Only source has retain + case is_map_key(<<"retain">>, Params0) of + %% Only source has retain + true -> <<"publisher">>; - _ -> + false -> <<"subscriber">> end, - Parms2 = maps:remove(<<"direction">>, Params), + Params1 = maps:remove(<<"direction">>, Params0), + Params = maps:remove(<<"local">>, Params1), + %% hidden; for backwards compatibility + LocalParams = maps:get(<<"local">>, Params1, #{}), DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), PoolSize = maps:get(<<"pool_size">>, ConnectorConfig, DefaultPoolSize), - Parms3 = maps:put(<<"pool_size">>, PoolSize, Parms2), ConnectorConfig2 = maps:remove(<<"pool_size">>, ConnectorConfig), LocalTopic = maps:get(<<"local_topic">>, ActionConfig, undefined), BridgeV1Conf0 = case {Direction, LocalTopic} of {<<"publisher">>, undefined} -> - #{<<"egress">> => Parms3}; + #{ + <<"egress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + }; {<<"publisher">>, LocalT} -> #{ <<"egress">> => - maps:merge( - Parms3, #{ - <<"local">> => - #{ - <<"topic">> => LocalT - } - } - ) + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => + maps:merge( + LocalParams, + #{<<"topic">> => LocalT} + ) + } }; {<<"subscriber">>, _} -> - #{<<"ingress">> => Parms3} + #{ + <<"ingress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + } end, BridgeV1Conf1 = maps:merge(BridgeV1Conf0, ConnectorConfig2), BridgeV1Conf2 = BridgeV1Conf1#{ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index b4c2b63ba..4cf092a60 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -56,10 +56,18 @@ fields("mqtt_publisher_action") -> ) ); fields(action_parameters) -> - Fields0 = emqx_bridge_mqtt_connector_schema:fields("egress"), - Fields1 = proplists:delete(pool_size, Fields0), - Fields2 = proplists:delete(local, Fields1), - Fields2; + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "egress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("egress_remote") + ]; fields(source) -> {mqtt, mk( @@ -71,8 +79,8 @@ fields(source) -> )}; fields("mqtt_subscriber_source") -> emqx_bridge_v2_schema:make_consumer_action_schema( - hoconsc:mk( - hoconsc:ref(?MODULE, ingress_parameters), + mk( + ref(?MODULE, ingress_parameters), #{ required => true, desc => ?DESC("source_parameters") @@ -80,10 +88,18 @@ fields("mqtt_subscriber_source") -> ) ); fields(ingress_parameters) -> - Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), - Fields1 = proplists:delete(pool_size, Fields0), - %% FIXME: should we make `local` hidden? - Fields1; + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "ingress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("ingress_remote") + ]; fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index 5569a826b..a0b3edfa7 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -108,11 +108,8 @@ source_config(Overrides0) -> <<"connector">> => <<"please override">>, <<"parameters">> => #{ - <<"remote">> => - #{ - <<"topic">> => <<"remote/topic">>, - <<"qos">> => 2 - } + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 }, <<"resource_opts">> => #{ <<"batch_size">> => 1, @@ -197,7 +194,7 @@ t_receive_via_rule(Config) -> Hookpoint = hookpoint(Config), RepublishTopic = <<"rep/t">>, RemoteTopic = emqx_utils_maps:deep_get( - [<<"parameters">>, <<"remote">>, <<"topic">>], + [<<"parameters">>, <<"topic">>], SourceConfig ), RuleOpts = #{ From 440a543a85e27977d357cc1bd707088f80792080 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jan 2024 13:56:17 -0300 Subject: [PATCH 24/28] docs: fix typo --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 8f36fd700..262ada984 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,7 +49,7 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). -%% only for testting/mocking +%% only for testing/mocking -export([supported_versions/1]). -define(BPAPI_NAME, emqx_bridge). From c6cd3adccbca63697035dc36ac914da61c113484 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 16 Jan 2024 11:51:53 +0100 Subject: [PATCH 25/28] refactor: fix type upgrade calls and move compatiblitly logic Some bridge V1 to V2 calls were wrong but did not seem to cause issues (perhaps due to locking test coverage). This commit also move compatibility logic from the API module to the emqx_bridge_v2 module where most of the compatibility logic exists. --- apps/emqx_bridge/src/emqx_bridge.erl | 4 ++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 6 +----- apps/emqx_bridge/src/emqx_bridge_resource.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 8 ++++++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 7df55c81c..e27748610 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -352,10 +352,10 @@ get_metrics(ActionType, Name) -> true -> case emqx_bridge_v2:bridge_v1_is_valid(ActionType, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(ActionType), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(ActionType), try ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( - ActionType, Name + BridgeV2Type, Name ), emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) catch diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 262ada984..7e929a233 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -554,11 +554,7 @@ schema("/bridges_probe") -> case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> try - ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( - BridgeType, BridgeName - ), - BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(ConfRootKey, BridgeV2Type, BridgeName), + ok = emqx_bridge_v2:bridge_v1_reset_metrics(BridgeType, BridgeName), ?NO_CONTENT catch error:Reason -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ec7a7431b..143956b5d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -147,7 +147,7 @@ reset_metrics(ResourceId) -> true -> case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(Type), emqx_bridge_v2:reset_metrics(BridgeV2Type, Name); false -> {error, not_bridge_v1_compatible} diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 32034f774..d76f737b5 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -135,6 +135,7 @@ bridge_v1_restart/2, bridge_v1_stop/2, bridge_v1_start/2, + bridge_v1_reset_metrics/2, %% For test cases only bridge_v1_remove/2, get_conf_root_key_if_only_one/2 @@ -1815,6 +1816,13 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> {error, not_bridge_v1_compatible} end. +bridge_v1_reset_metrics(BridgeV1Type, BridgeName) -> + BridgeV2Type = bridge_v1_type_to_bridge_v2_type(BridgeV1Type), + ConfRootKey = get_conf_root_key_if_only_one( + BridgeV2Type, BridgeName + ), + ok = reset_metrics(ConfRootKey, BridgeV2Type, BridgeName). + %%==================================================================== %% Misc helper functions %%==================================================================== From 60fab6ee45fef2f3087e26e9208b12548221f5d8 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 16 Jan 2024 12:20:35 +0100 Subject: [PATCH 26/28] refactor: attempt to improve function names --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index d76f737b5..b69882080 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1255,7 +1255,7 @@ bridge_v1_list_and_transform() -> BridgesFromActions1 ++ BridgesFromSources1. bridge_v1_lookup_and_transform(ActionType, Name) -> - case lookup_actions_or_sources(ActionType, Name) of + case lookup_in_actions_or_sources(ActionType, Name) of {ok, ConfRootKey, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), @@ -1287,7 +1287,7 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> Error end. -lookup_actions_or_sources(ActionType, Name) -> +lookup_in_actions_or_sources(ActionType, Name) -> case lookup(?ROOT_KEY_ACTIONS, ActionType, Name) of {error, not_found} -> case lookup(?ROOT_KEY_SOURCES, ActionType, Name) of @@ -1356,7 +1356,7 @@ bridge_v1_lookup_and_transform_helper( lookup_conf(Type, Name) -> lookup_conf(?ROOT_KEY_ACTIONS, Type, Name). -lookup_conf_if_one_of_sources_actions(Type, Name) -> +lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(Type, Name) -> LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, Type, Name), LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, Type, Name), case {LookUpConfActions, LookUpConfSources} of @@ -1409,7 +1409,7 @@ lookup_conf(RootName, Type, Name) -> bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), %% Check if the bridge v2 exists - case lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) of + case lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, @@ -1636,7 +1636,7 @@ bridge_v1_remove(BridgeV1Type, BridgeName) -> bridge_v1_remove( ActionType, BridgeName, - lookup_conf_if_one_of_sources_actions(ActionType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(ActionType, BridgeName) ). bridge_v1_remove( @@ -1665,7 +1665,7 @@ bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) -> BridgeV2Type, BridgeName, RemoveDeps, - lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) ). %% Bridge v1 delegated-removal in 3 steps: @@ -1760,7 +1760,7 @@ bridge_v1_enable_disable(Action, BridgeType, BridgeName) -> Action, BridgeType, BridgeName, - lookup_conf_if_one_of_sources_actions(BridgeType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeType, BridgeName) ); false -> {error, not_bridge_v1_compatible} @@ -1808,7 +1808,7 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> ConfRootKey, BridgeV2Type, Name, - lookup_conf_if_one_of_sources_actions(BridgeV2Type, Name), + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ); From a8f9e5676f99eedd4101ae7945fb4f721c3547a3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 10:24:19 -0300 Subject: [PATCH 27/28] docs(mqtt_bridge): add API examples --- apps/emqx_bridge/src/emqx_bridge_api.erl | 8 ++- .../src/schema/emqx_bridge_v2_schema.erl | 4 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 51 +++++++++++++++++-- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 7e929a233..f53503b86 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,6 +49,9 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). +%% used by actions/sources schema +-export([mqtt_v1_example/1]). + %% only for testing/mocking -export([supported_versions/1]). @@ -181,7 +184,7 @@ bridge_info_examples(Method) -> }, <<"mqtt_example">> => #{ summary => <<"MQTT Bridge">>, - value => info_example(mqtt, Method) + value => mqtt_v1_example(Method) } }, emqx_enterprise_bridge_examples(Method) @@ -194,6 +197,9 @@ emqx_enterprise_bridge_examples(Method) -> emqx_enterprise_bridge_examples(_Method) -> #{}. -endif. +mqtt_v1_example(Method) -> + info_example(mqtt, Method). + info_example(Type, Method) -> maps:merge( info_example_basic(Type), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index ec9314fd2..35616ae7e 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -176,7 +176,7 @@ source_values(Method, SourceType, ConnectorType, SourceValues) -> description => <<"My example ", SourceTypeBin/binary, " source">>, connector => <>, resource_opts => #{ - health_check_interval => "30s" + health_check_interval => <<"30s">> } }, [ @@ -192,7 +192,7 @@ sources_examples(Method) -> end, Fun = fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + ConnectorExamples = erlang:apply(Module, source_examples, [Method]), lists:foldl(MergeFun, Examples, ConnectorExamples) end, SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_sources()], diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 4cf092a60..c05566234 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -24,6 +24,7 @@ -export([ bridge_v2_examples/1, + source_examples/1, conn_bridge_examples/1 ]). @@ -148,12 +149,54 @@ desc("mqtt_subscriber_source") -> desc(_) -> undefined. -bridge_v2_examples(_Method) -> +bridge_v2_examples(Method) -> [ - #{} + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, + _ActionType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2, + retain => false, + payload => <<"${.payload}">> + } + } + ) + } + } ]. -conn_bridge_examples(_Method) -> +source_examples(Method) -> [ - #{} + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Subscriber Source">>, + value => emqx_bridge_v2_schema:source_values( + Method, + _SourceType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2 + } + } + ) + } + } + ]. + +conn_bridge_examples(Method) -> + [ + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_api:mqtt_v1_example(Method) + } + } ]. From 2a41cad54feb20a83dcea53adacaa330fb2596bc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 10:53:27 -0300 Subject: [PATCH 28/28] fix(sources): remote irrelevant `resource_opts` fields for sources Since they don't use buffer workers, they shouldn't have buffer-related sub-fields. --- .../src/schema/emqx_bridge_v2_schema.erl | 92 ++++++++++++++----- .../test/emqx_bridge_v2_api_SUITE.erl | 12 +-- .../emqx_bridge/test/emqx_bridge_v2_tests.erl | 4 +- apps/emqx_bridge_es/src/emqx_bridge_es.erl | 2 +- .../src/emqx_bridge_http_schema.erl | 2 +- .../src/emqx_bridge_iotdb.erl | 2 +- .../src/emqx_bridge_kafka.erl | 2 +- .../src/emqx_bridge_mongodb.erl | 2 +- .../emqx_bridge_mqtt_pubsub_action_info.erl | 2 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 6 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 12 +-- .../src/emqx_bridge_redis_schema.erl | 2 +- 12 files changed, 84 insertions(+), 56 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 35616ae7e..5b9500156 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -53,7 +53,8 @@ -export([action_types/0, action_types_sc/0]). -export([source_types/0, source_types_sc/0]). --export([resource_opts_fields/0, resource_opts_fields/1]). +-export([action_resource_opts_fields/0, action_resource_opts_fields/1]). +-export([source_resource_opts_fields/0, source_resource_opts_fields/1]). -export([ api_fields/3 @@ -63,7 +64,8 @@ make_producer_action_schema/1, make_producer_action_schema/2, make_consumer_action_schema/1, make_consumer_action_schema/2, top_level_common_action_keys/0, - project_to_actions_resource_opts/1 + project_to_actions_resource_opts/1, + project_to_sources_resource_opts/1 ]). -export([actions_convert_from_connectors/1]). @@ -317,8 +319,10 @@ fields(actions) -> registered_schema_fields_actions(); fields(sources) -> registered_schema_fields_sources(); -fields(resource_opts) -> - resource_opts_fields(_Overrides = []). +fields(action_resource_opts) -> + action_resource_opts_fields(_Overrides = []); +fields(source_resource_opts) -> + source_resource_opts_fields(_Overrides = []). registered_schema_fields_actions() -> [ @@ -336,7 +340,9 @@ desc(actions) -> ?DESC("desc_bridges_v2"); desc(sources) -> ?DESC("desc_sources"); -desc(resource_opts) -> +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(source_resource_opts) -> ?DESC(emqx_resource_schema, "resource_opts"); desc(_) -> undefined. @@ -357,10 +363,13 @@ source_types() -> source_types_sc() -> hoconsc:enum(source_types()). -resource_opts_fields() -> - resource_opts_fields(_Overrides = []). +action_resource_opts_fields() -> + action_resource_opts_fields(_Overrides = []). -common_resource_opts_subfields() -> +source_resource_opts_fields() -> + source_resource_opts_fields(_Overrides = []). + +common_action_resource_opts_subfields() -> [ batch_size, batch_time, @@ -376,11 +385,27 @@ common_resource_opts_subfields() -> worker_pool_size ]. -common_resource_opts_subfields_bin() -> - lists:map(fun atom_to_binary/1, common_resource_opts_subfields()). +common_source_resource_opts_subfields() -> + [ + health_check_interval, + resume_interval + ]. -resource_opts_fields(Overrides) -> - ActionROFields = common_resource_opts_subfields(), +common_action_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_action_resource_opts_subfields()). + +common_source_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_source_resource_opts_subfields()). + +action_resource_opts_fields(Overrides) -> + ActionROFields = common_action_resource_opts_subfields(), + lists:filter( + fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, + emqx_resource_schema:create_opts(Overrides) + ). + +source_resource_opts_fields(Overrides) -> + ActionROFields = common_source_resource_opts_subfields(), lists:filter( fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, emqx_resource_schema:create_opts(Overrides) @@ -404,16 +429,34 @@ make_producer_action_schema(ActionParametersRef) -> make_producer_action_schema(ActionParametersRef, _Opts = #{}). make_producer_action_schema(ActionParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, action_resource_opts)), [ {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})} - | make_consumer_action_schema(ActionParametersRef, Opts) - ]. + | common_schema(ActionParametersRef, Opts) + ] ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. -make_consumer_action_schema(ActionParametersRef) -> - make_consumer_action_schema(ActionParametersRef, _Opts = #{}). +make_consumer_action_schema(ParametersRef) -> + make_consumer_action_schema(ParametersRef, _Opts = #{}). -make_consumer_action_schema(ActionParametersRef, Opts) -> - ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, resource_opts)), +make_consumer_action_schema(ParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, source_resource_opts)), + common_schema(ParametersRef, Opts) ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. + +common_schema(ParametersRef, _Opts) -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {connector, @@ -421,16 +464,15 @@ make_consumer_action_schema(ActionParametersRef, Opts) -> desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, {description, emqx_schema:description_schema()}, - {parameters, ActionParametersRef}, - {resource_opts, - mk(ResourceOptsRef, #{ - default => #{}, - desc => ?DESC(emqx_resource_schema, "resource_opts") - })} + {parameters, ParametersRef} ]. project_to_actions_resource_opts(OldResourceOpts) -> - Subfields = common_resource_opts_subfields_bin(), + Subfields = common_action_resource_opts_subfields_bin(), + maps:with(Subfields, OldResourceOpts). + +project_to_sources_resource_opts(OldResourceOpts) -> + Subfields = common_source_resource_opts_subfields_bin(), maps:with(Subfields, OldResourceOpts). actions_convert_from_connectors(RawConf = #{<<"actions">> := Actions}) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index fabaadb92..fc9c9573f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -455,18 +455,8 @@ source_config_base() -> <<"qos">> => 2 }, <<"resource_opts">> => #{ - <<"batch_size">> => 1, - <<"batch_time">> => <<"0ms">>, - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, <<"health_check_interval">> => <<"15s">>, - <<"inflight_window">> => 100, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"45s">>, - <<"resume_interval">> => <<"15s">>, - <<"worker_pool_size">> => <<"1">> + <<"resume_interval">> => <<"15s">> } }. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index c64b1f2cb..a6c66c609 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -48,7 +48,9 @@ resource_opts_union_connector_actions_test() -> %% consciouly between connector and actions, in particular when/if we introduce new %% fields there. AllROFields = non_deprecated_fields(emqx_resource_schema:create_opts([])), - ActionROFields = non_deprecated_fields(emqx_bridge_v2_schema:resource_opts_fields()), + ActionROFields = non_deprecated_fields( + emqx_bridge_v2_schema:action_resource_opts_fields() + ), ConnectorROFields = non_deprecated_fields(emqx_connector_schema:resource_opts_fields()), UnionROFields = lists:usort(ConnectorROFields ++ ActionROFields), ?assertEqual( diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index b575f32ed..20a768d53 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -52,7 +52,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_create) -> [ diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index a4d956d78..b8968e82c 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -105,7 +105,7 @@ fields(action_resource_opts) -> UnsupportedOpts = [batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields("parameters_opts") -> [ diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 4be7feb19..c33ea757b 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -70,7 +70,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_parameters) -> [ diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index d74ff40a1..235bc4783 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -552,7 +552,7 @@ fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(resource_opts) -> SupportedFields = [health_check_interval], - CreationOpts = emqx_bridge_v2_schema:resource_opts_fields(), + CreationOpts = emqx_bridge_v2_schema:action_resource_opts_fields(), lists:filter(fun({Field, _}) -> lists:member(Field, SupportedFields) end, CreationOpts); fields(action_field) -> {kafka_producer, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index e8eb93624..ba7be1ec4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -97,7 +97,7 @@ fields(action_parameters) -> fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{ importance => ?IMPORTANCE_HIDDEN, converter => fun(_, _) -> 1 end, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index cf7a5bc04..407a25118 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -116,7 +116,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(source_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index c05566234..05b2d6d3a 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -105,8 +105,10 @@ fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); +fields(source_resource_opts) -> + emqx_bridge_v2_schema:source_resource_opts_fields(); fields(Field) when Field == "get_bridge_v2"; Field == "post_bridge_v2"; @@ -132,6 +134,8 @@ desc("config") -> ?DESC("desc_config"); desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); +desc(source_resource_opts) -> + ?DESC(emqx_resource_schema, "creation_opts"); desc(action_parameters) -> ?DESC(action_parameters); desc(ingress_parameters) -> diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index a0b3edfa7..3e5471d55 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -112,18 +112,8 @@ source_config(Overrides0) -> <<"qos">> => 2 }, <<"resource_opts">> => #{ - <<"batch_size">> => 1, - <<"batch_time">> => <<"0ms">>, - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, <<"health_check_interval">> => <<"15s">>, - <<"inflight_window">> => 100, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"45s">>, - <<"resume_interval">> => <<"15s">>, - <<"worker_pool_size">> => <<"1">> + <<"resume_interval">> => <<"15s">> } }, maps:merge(CommonConfig, Overrides). diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index 9373fe8bd..adda91f37 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -76,7 +76,7 @@ fields(redis_action) -> [ResOpts] = emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts), lists:keyreplace(resource_opts, 1, Schema, ResOpts); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{desc => ?DESC(batch_size)}}, {batch_time, #{desc => ?DESC(batch_time)}} ]);