diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 9721a7f2f..9bd824242 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -8,6 +8,7 @@ {emqx_bridge,3}. {emqx_bridge,4}. {emqx_bridge,5}. +{emqx_bridge,6}. {emqx_broker,1}. {emqx_cm,1}. {emqx_cm,2}. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 3ffcb1a6a..81314ce23 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1387,29 +1387,40 @@ matrix_to_groups(Module, Cases) -> Cases ). -add_case_matrix(Module, Case, Acc0) -> - {RootGroup, Matrix} = Module:Case(matrix), +add_case_matrix(Module, TestCase, Acc0) -> + {MaybeRootGroup, Matrix} = + case Module:TestCase(matrix) of + {RootGroup0, Matrix0} -> + {RootGroup0, Matrix0}; + Matrix0 -> + {undefined, Matrix0} + end, lists:foldr( fun(Row, Acc) -> - add_group([RootGroup | Row], Acc, Case) + case MaybeRootGroup of + undefined -> + add_group(Row, Acc, TestCase); + RootGroup -> + add_group([RootGroup | Row], Acc, TestCase) + end end, Acc0, Matrix ). -add_group([], Acc, Case) -> - case lists:member(Case, Acc) of +add_group([], Acc, TestCase) -> + case lists:member(TestCase, Acc) of true -> Acc; false -> - [Case | Acc] + [TestCase | Acc] end; -add_group([Name | More], Acc, Cases) -> +add_group([Name | More], Acc, TestCases) -> case lists:keyfind(Name, 1, Acc) of false -> - [{Name, [], add_group(More, [], Cases)} | Acc]; + [{Name, [], add_group(More, [], TestCases)} | Acc]; {Name, [], SubGroup} -> - New = {Name, [], add_group(More, SubGroup, Cases)}, + New = {Name, [], add_group(More, SubGroup, TestCases)}, lists:keystore(Name, 1, Acc, New) end. diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 5ce60fe6c..d80050191 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -26,7 +26,10 @@ bridge_v1_type_to_action_type/1, bridge_v1_type_name/1, is_action_type/1, - registered_schema_modules/0, + is_source/1, + is_action/1, + registered_schema_modules_actions/0, + registered_schema_modules_sources/0, connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, bridge_v1_config_to_connector_config/2, @@ -51,19 +54,26 @@ ConnectorConfig :: map(), ActionConfig :: map() ) -> map(). %% Define this if the automatic config upgrade is not enough for the connector. --callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> map(). +-callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> + map() | {ConnectorTypeName :: atom(), map()}. %% Define this if the automatic config upgrade is not enough for the bridge. %% If you want to make use of the automatic config upgrade, you can call %% emqx_action_info:transform_bridge_v1_config_to_action_config/4 in your %% implementation and do some adjustments on the result. -callback bridge_v1_config_to_action_config(BridgeV1Config :: map(), ConnectorName :: binary()) -> - map(). + map() | {source | action, ActionTypeName :: atom(), map()} | 'none'. +-callback is_source() -> + boolean(). +-callback is_action() -> + boolean(). -optional_callbacks([ bridge_v1_type_name/0, connector_action_config_to_bridge_v1_config/2, bridge_v1_config_to_connector_config/1, - bridge_v1_config_to_action_config/2 + bridge_v1_config_to_action_config/2, + is_source/0, + is_action/0 ]). %% ==================================================================== @@ -96,7 +106,10 @@ hard_coded_action_info_modules_ee() -> -endif. hard_coded_action_info_modules_common() -> - [emqx_bridge_http_action_info]. + [ + emqx_bridge_http_action_info, + emqx_bridge_mqtt_pubsub_action_info + ]. hard_coded_action_info_modules() -> hard_coded_action_info_modules_common() ++ hard_coded_action_info_modules_ee(). @@ -178,10 +191,33 @@ is_action_type(Type) -> _ -> true end. -registered_schema_modules() -> +%% Returns true if the action is an ingress action, false otherwise. +is_source(Bin) when is_binary(Bin) -> + is_source(binary_to_existing_atom(Bin)); +is_source(Type) -> + ActionInfoMap = info_map(), + IsSourceMap = maps:get(is_source, ActionInfoMap), + maps:get(Type, IsSourceMap, false). + +%% Returns true if the action is an egress action, false otherwise. +is_action(Bin) when is_binary(Bin) -> + is_action(binary_to_existing_atom(Bin)); +is_action(Type) -> + ActionInfoMap = info_map(), + IsActionMap = maps:get(is_action, ActionInfoMap), + maps:get(Type, IsActionMap, true). + +registered_schema_modules_actions() -> InfoMap = info_map(), Schemas = maps:get(action_type_to_schema_module, InfoMap), - maps:to_list(Schemas). + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_action(Type)]. + +registered_schema_modules_sources() -> + InfoMap = info_map(), + Schemas = maps:get(action_type_to_schema_module, InfoMap), + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_source(Type)]. connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), @@ -293,7 +329,9 @@ initial_info_map() -> action_type_to_bridge_v1_type => #{}, action_type_to_connector_type => #{}, action_type_to_schema_module => #{}, - action_type_to_info_module => #{} + action_type_to_info_module => #{}, + is_source => #{}, + is_action => #{} }. get_info_map(Module) -> @@ -312,6 +350,20 @@ get_info_map(Module) -> false -> {ActionType, [ActionType]} end, + IsIngress = + case erlang:function_exported(Module, is_source, 0) of + true -> + Module:is_source(); + false -> + false + end, + IsEgress = + case erlang:function_exported(Module, is_action, 0) of + true -> + Module:is_action(); + false -> + true + end, #{ action_type_names => lists:foldl( @@ -351,5 +403,11 @@ get_info_map(Module) -> end, #{ActionType => Module}, BridgeV1Types - ) + ), + is_source => #{ + ActionType => IsIngress + }, + is_action => #{ + ActionType => IsEgress + } }. diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d26a44a1d..e27748610 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -347,18 +347,26 @@ lookup(Type, Name, RawConf) -> }} end. -get_metrics(Type, Name) -> - case emqx_bridge_v2:is_bridge_v2_type(Type) of +get_metrics(ActionType, Name) -> + case emqx_bridge_v2:is_bridge_v2_type(ActionType) of true -> - case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of + case emqx_bridge_v2:bridge_v1_is_valid(ActionType, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), - emqx_bridge_v2:get_metrics(BridgeV2Type, Name); + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(ActionType), + try + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( + BridgeV2Type, Name + ), + emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) + catch + error:Reason -> + {error, Reason} + end; false -> {error, not_bridge_v1_compatible} end; false -> - emqx_resource:get_metrics(emqx_bridge_resource:resource_id(Type, Name)) + emqx_resource:get_metrics(emqx_bridge_resource:resource_id(ActionType, Name)) end. maybe_upgrade(mqtt, Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index e1cd03ac2..f53503b86 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,6 +49,14 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). +%% used by actions/sources schema +-export([mqtt_v1_example/1]). + +%% only for testing/mocking +-export([supported_versions/1]). + +-define(BPAPI_NAME, emqx_bridge). + -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). @@ -176,7 +184,7 @@ bridge_info_examples(Method) -> }, <<"mqtt_example">> => #{ summary => <<"MQTT Bridge">>, - value => info_example(mqtt, Method) + value => mqtt_v1_example(Method) } }, emqx_enterprise_bridge_examples(Method) @@ -189,6 +197,9 @@ emqx_enterprise_bridge_examples(Method) -> emqx_enterprise_bridge_examples(_Method) -> #{}. -endif. +mqtt_v1_example(Method) -> + info_example(mqtt, Method). + info_example(Type, Method) -> maps:merge( info_example_basic(Type), @@ -548,9 +559,13 @@ schema("/bridges_probe") -> Id, case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(BridgeV2Type, BridgeName), - ?NO_CONTENT; + try + ok = emqx_bridge_v2:bridge_v1_reset_metrics(BridgeType, BridgeName), + ?NO_CONTENT + catch + error:Reason -> + ?BAD_REQUEST(Reason) + end; false -> ok = emqx_bridge_resource:reset_metrics( emqx_bridge_resource:resource_id(BridgeType, BridgeName) @@ -1094,18 +1109,18 @@ maybe_try_restart(_, _, _) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> case lists:member(Node, mria:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. do_bpapi_call_vsn(SupportedVersion, Call, Args) -> - case lists:member(SupportedVersion, supported_versions(Call)) of + case lists:member(SupportedVersion, ?MODULE:supported_versions(Call)) of true -> apply(emqx_bridge_proto_v4, Call, Args); false -> @@ -1117,10 +1132,15 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2, 3, 4, 5]; -supported_versions(start_bridges_to_all_nodes) -> [2, 3, 4, 5]; -supported_versions(get_metrics_from_all_nodes) -> [4, 5]; -supported_versions(_Call) -> [1, 2, 3, 4, 5]. +supported_versions(start_bridge_to_node) -> bpapi_version_range(2, latest); +supported_versions(start_bridges_to_all_nodes) -> bpapi_version_range(2, latest); +supported_versions(get_metrics_from_all_nodes) -> bpapi_version_range(4, latest); +supported_versions(_Call) -> bpapi_version_range(1, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). redact(Term) -> emqx_utils:redact(Term). diff --git a/apps/emqx_bridge/src/emqx_bridge_lib.erl b/apps/emqx_bridge/src/emqx_bridge_lib.erl index 9386a38d3..7f74dfb2d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_lib.erl +++ b/apps/emqx_bridge/src/emqx_bridge_lib.erl @@ -82,7 +82,8 @@ external_ids(Type, Name) -> get_conf(BridgeType, BridgeName) -> case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - emqx_conf:get_raw([actions, BridgeType, BridgeName]); + ConfRootName = emqx_bridge_v2:get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_conf:get_raw([ConfRootName, BridgeType, BridgeName]); false -> undefined end. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 0a870abb8..143956b5d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -23,6 +23,7 @@ bridge_to_resource_type/1, resource_id/1, resource_id/2, + resource_id/3, bridge_id/2, parse_bridge_id/1, parse_bridge_id/2, @@ -62,6 +63,9 @@ ?IS_BI_DIR_BRIDGE(TYPE) ). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(BridgeType) when is_binary(BridgeType) -> bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8)); @@ -85,11 +89,21 @@ bridge_impl_module(_BridgeType) -> undefined. -endif. resource_id(BridgeId) when is_binary(BridgeId) -> + resource_id_for_kind(?ROOT_KEY_ACTIONS, BridgeId). + +resource_id(BridgeType, BridgeName) -> + resource_id(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +resource_id(ConfRootKey, BridgeType, BridgeName) -> + BridgeId = bridge_id(BridgeType, BridgeName), + resource_id_for_kind(ConfRootKey, BridgeId). + +resource_id_for_kind(ConfRootKey, BridgeId) when is_binary(BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, _Name] -> case emqx_bridge_v2:is_bridge_v2_type(Type) of true -> - emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(BridgeId); + emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId); false -> <<"bridge:", BridgeId/binary>> end; @@ -97,10 +111,6 @@ resource_id(BridgeId) when is_binary(BridgeId) -> invalid_data(<<"should be of pattern {type}:{name}, but got ", BridgeId/binary>>) end. -resource_id(BridgeType, BridgeName) -> - BridgeId = bridge_id(BridgeType, BridgeName), - resource_id(BridgeId). - bridge_id(BridgeType, BridgeName) -> Name = bin(BridgeName), Type = bin(BridgeType), @@ -137,7 +147,7 @@ reset_metrics(ResourceId) -> true -> case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(Type), emqx_bridge_v2:reset_metrics(BridgeV2Type, Name); false -> {error, not_bridge_v1_compatible} diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 723808919..b69882080 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -26,7 +26,8 @@ %% Note: this is strange right now, because it lives in `emqx_bridge_v2', but it shall be %% refactored into a new module/application with appropriate name. --define(ROOT_KEY, actions). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). %% Loading and unloading config when EMQX starts and stops -export([ @@ -38,28 +39,39 @@ -export([ list/0, + list/1, lookup/2, + lookup/3, create/3, + create/4, %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies remove/2, + remove/3, %% The following is the remove function that is called by the HTTP API %% It also checks for rule action dependencies and optionally removes %% them - check_deps_and_remove/3 + check_deps_and_remove/3, + check_deps_and_remove/4 ]). +-export([lookup_action/2, lookup_source/2]). %% Operations -export([ disable_enable/3, + disable_enable/4, health_check/2, send_message/4, query/4, start/2, + start/3, reset_metrics/2, + reset_metrics/3, create_dry_run/2, - get_metrics/2 + create_dry_run/3, + get_metrics/2, + get_metrics/3 ]). %% On message publish hook (for local_topics) @@ -80,6 +92,7 @@ id/2, id/3, bridge_v1_is_valid/2, + bridge_v1_is_valid/3, extract_connector_id_from_bridge_v2_id/1 ]). @@ -117,12 +130,15 @@ %% Exception from the naming convention: bridge_v2_type_to_bridge_v1_type/2, bridge_v1_id_to_connector_resource_id/1, + bridge_v1_id_to_connector_resource_id/2, bridge_v1_enable_disable/3, bridge_v1_restart/2, bridge_v1_stop/2, bridge_v1_start/2, + bridge_v1_reset_metrics/2, %% For test cases only - bridge_v1_remove/2 + bridge_v1_remove/2, + get_conf_root_key_if_only_one/2 ]). %%==================================================================== @@ -142,6 +158,10 @@ -type bridge_v2_type() :: binary() | atom() | [byte()]. -type bridge_v2_name() :: binary() | atom() | [byte()]. +-type root_cfg_key() :: ?ROOT_KEY_ACTIONS | ?ROOT_KEY_SOURCES. + +-export_type([root_cfg_key/0]). + %%==================================================================== %%==================================================================== @@ -151,19 +171,22 @@ %%==================================================================== load() -> - load_bridges(), + load_bridges(?ROOT_KEY_ACTIONS), + load_bridges(?ROOT_KEY_SOURCES), load_message_publish_hook(), ok = emqx_config_handler:add_handler(config_key_path_leaf(), emqx_bridge_v2), ok = emqx_config_handler:add_handler(config_key_path(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_leaf_sources(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_sources(), emqx_bridge_v2), ok. -load_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +load_bridges(RootName) -> + Bridges = emqx:get_config([RootName], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - install_bridge_v2(Type, Name, BridgeConf) + install_bridge_v2(RootName, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -172,19 +195,20 @@ load_bridges() -> ). unload() -> - unload_bridges(), + unload_bridges(?ROOT_KEY_ACTIONS), + unload_bridges(?ROOT_KEY_SOURCES), unload_message_publish_hook(), emqx_conf:remove_handler(config_key_path()), emqx_conf:remove_handler(config_key_path_leaf()), ok. -unload_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +unload_bridges(ConfRooKey) -> + Bridges = emqx:get_config([ConfRooKey], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - uninstall_bridge_v2(Type, Name, BridgeConf) + uninstall_bridge_v2(ConfRooKey, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -198,7 +222,18 @@ unload_bridges() -> -spec lookup(bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(Type, Name) -> - case emqx:get_raw_config([?ROOT_KEY, Type, Name], not_found) of + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_action(Type, Name) -> + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_source(Type, Name) -> + lookup(?ROOT_KEY_SOURCES, Type, Name). + +-spec lookup(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> + {ok, bridge_v2_info()} | {error, not_found}. +lookup(ConfRootName, Type, Name) -> + case emqx:get_raw_config([ConfRootName, Type, Name], not_found) of not_found -> {error, not_found}; #{<<"connector">> := BridgeConnector} = RawConf -> @@ -218,7 +253,7 @@ lookup(Type, Name) -> %% Find the Bridge V2 status from the ConnectorData ConnectorStatus = maps:get(status, ConnectorData, undefined), Channels = maps:get(added_channels, ConnectorData, #{}), - BridgeV2Id = id(Type, Name, BridgeConnector), + BridgeV2Id = id_with_root_name(ConfRootName, Type, Name, BridgeConnector), ChannelStatus = maps:get(BridgeV2Id, Channels, undefined), {DisplayBridgeV2Status, ErrorMsg} = case {ChannelStatus, ConnectorStatus} of @@ -245,20 +280,30 @@ lookup(Type, Name) -> -spec list() -> [bridge_v2_info()] | {error, term()}. list() -> - list_with_lookup_fun(fun lookup/2). + list_with_lookup_fun(?ROOT_KEY_ACTIONS, fun lookup/2). + +list(ConfRootKey) -> + LookupFun = fun(Type, Name) -> + lookup(ConfRootKey, Type, Name) + end, + list_with_lookup_fun(ConfRootKey, LookupFun). -spec create(bridge_v2_type(), bridge_v2_name(), map()) -> {ok, emqx_config:update_result()} | {error, any()}. create(BridgeType, BridgeName, RawConf) -> + create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, RawConf). + +create(ConfRootKey, BridgeType, BridgeName, RawConf) -> ?SLOG(debug, #{ bridge_action => create, bridge_version => 2, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_utils:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf), + root_key_path => ConfRootKey }), emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], RawConf, #{override_to => cluster} ). @@ -267,6 +312,9 @@ create(BridgeType, BridgeName, RawConf) -> remove(BridgeType, BridgeName) -> %% NOTE: This function can cause broken references from rules but it is only %% called directly from test cases. + remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +remove(ConfRootKey, BridgeType, BridgeName) -> ?SLOG(debug, #{ bridge_action => remove, bridge_version => 2, @@ -275,7 +323,7 @@ remove(BridgeType, BridgeName) -> }), case emqx_conf:remove( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], #{override_to => cluster} ) of @@ -285,6 +333,11 @@ remove(BridgeType, BridgeName) -> -spec check_deps_and_remove(bridge_v2_type(), bridge_v2_name(), boolean()) -> ok | {error, any()}. check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> + check_deps_and_remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, AlsoDeleteActions). + +-spec check_deps_and_remove(root_cfg_key(), bridge_v2_type(), bridge_v2_name(), boolean()) -> + ok | {error, any()}. +check_deps_and_remove(ConfRooKey, BridgeType, BridgeName, AlsoDeleteActions) -> AlsoDelete = case AlsoDeleteActions of true -> [rule_actions]; @@ -298,7 +351,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> ) of ok -> - remove(BridgeType, BridgeName); + remove(ConfRooKey, BridgeType, BridgeName); {error, Reason} -> {error, Reason} end. @@ -307,7 +360,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> %% Helpers for CRUD API %%-------------------------------------------------------------------- -list_with_lookup_fun(LookupFun) -> +list_with_lookup_fun(ConfRootName, LookupFun) -> maps:fold( fun(Type, NameAndConf, Bridges) -> maps:fold( @@ -330,21 +383,24 @@ list_with_lookup_fun(LookupFun) -> ) end, [], - emqx:get_raw_config([?ROOT_KEY], #{}) + emqx:get_raw_config([ConfRootName], #{}) ). install_bridge_v2( + _RootName, _BridgeType, _BridgeName, #{enable := false} ) -> ok; install_bridge_v2( + RootName, BridgeV2Type, BridgeName, Config ) -> install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, combine_connector_and_bridge_v2_config( @@ -355,6 +411,7 @@ install_bridge_v2( ). install_bridge_v2_helper( + _RootName, _BridgeV2Type, _BridgeName, {error, Reason} = Error @@ -362,11 +419,12 @@ install_bridge_v2_helper( ?SLOG(warning, Reason), Error; install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(RootName, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), %% Create metrics for Bridge V2 ok = emqx_resource:create_metrics(BridgeV2Id), @@ -388,18 +446,45 @@ install_bridge_v2_helper( ConnectorId = emqx_connector_resource:resource_id( connector_type(BridgeV2Type), ConnectorName ), - ConfigWithTypeAndName = Config#{ - bridge_type => bin(BridgeV2Type), - bridge_name => bin(BridgeName) - }, emqx_resource_manager:add_channel( ConnectorId, BridgeV2Id, - ConfigWithTypeAndName + augment_channel_config( + RootName, + BridgeV2Type, + BridgeName, + Config + ) ), ok. +augment_channel_config( + ConfigRoot, + BridgeV2Type, + BridgeName, + Config +) -> + AugmentedConf = Config#{ + config_root => ConfigRoot, + bridge_type => bin(BridgeV2Type), + bridge_name => bin(BridgeName) + }, + case emqx_action_info:is_source(BridgeV2Type) of + true -> + BId = emqx_bridge_resource:bridge_id(BridgeV2Type, BridgeName), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + SourceHookpoint = source_hookpoint(BId), + HookPoints = [BridgeHookpoint, SourceHookpoint], + AugmentedConf#{hookpoints => HookPoints}; + false -> + AugmentedConf + end. + +source_hookpoint(BridgeId) -> + <<"$sources/", (bin(BridgeId))/binary>>. + uninstall_bridge_v2( + _ConfRootKey, _BridgeType, _BridgeName, #{enable := false} @@ -407,11 +492,12 @@ uninstall_bridge_v2( %% Already not installed ok; uninstall_bridge_v2( + ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), ok = emqx_resource_buffer_worker_sup:stop_workers(BridgeV2Id, CreationOpts), ok = emqx_resource:clear_metrics(BridgeV2Id), @@ -460,8 +546,11 @@ combine_connector_and_bridge_v2_config( -spec disable_enable(disable | enable, bridge_v2_type(), bridge_v2_name()) -> {ok, any()} | {error, any()}. disable_enable(Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> + disable_enable(?ROOT_KEY_ACTIONS, Action, BridgeType, BridgeName). + +disable_enable(ConfRootKey, Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], Action, #{override_to => cluster} ). @@ -473,30 +562,37 @@ disable_enable(Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) - %% is something else than connected after starting the connector or if an %% error occurred when the connector was started. -spec start(term(), term()) -> ok | {error, Reason :: term()}. -start(BridgeV2Type, Name) -> +start(ActionOrSourceType, Name) -> + start(?ROOT_KEY_ACTIONS, ActionOrSourceType, Name). + +-spec start(root_cfg_key(), term(), term()) -> ok | {error, Reason :: term()}. +start(ConfRootKey, BridgeV2Type, Name) -> ConnectorOpFun = fun(ConnectorType, ConnectorName) -> emqx_connector_resource:start(ConnectorType, ConnectorName) end, - connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, true). + connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, true). -connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> +connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf(ConfRootKey, BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ). connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, - {error, bridge_not_found} = Error, + {error, _} = Error, _ConnectorOpFun, _DoHealthCheck ) -> Error; connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, #{enable := false}, @@ -505,6 +601,7 @@ connector_operation_helper_with_conf( ) -> ok; connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, #{connector := ConnectorName}, @@ -519,7 +616,7 @@ connector_operation_helper_with_conf( {true, {error, Reason}} -> {error, Reason}; {true, ok} -> - case health_check(BridgeV2Type, Name) of + case health_check(ConfRootKey, BridgeV2Type, Name) of #{status := connected} -> ok; {error, Reason} -> @@ -536,14 +633,17 @@ connector_operation_helper_with_conf( -spec reset_metrics(bridge_v2_type(), bridge_v2_name()) -> ok | {error, not_found}. reset_metrics(Type, Name) -> - reset_metrics_helper(Type, Name, lookup_conf(Type, Name)). + reset_metrics(?ROOT_KEY_ACTIONS, Type, Name). -reset_metrics_helper(_Type, _Name, #{enable := false}) -> +reset_metrics(ConfRootKey, Type, Name) -> + reset_metrics_helper(ConfRootKey, Type, Name, lookup_conf(ConfRootKey, Type, Name)). + +reset_metrics_helper(_ConfRootKey, _Type, _Name, #{enable := false}) -> ok; -reset_metrics_helper(BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), +reset_metrics_helper(ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, BridgeV2Id); -reset_metrics_helper(_, _, _) -> +reset_metrics_helper(_, _, _, _) -> {error, not_found}. get_query_mode(BridgeV2Type, Config) -> @@ -599,7 +699,10 @@ send_message(BridgeType, BridgeName, Message, QueryOpts0) -> -spec health_check(BridgeType :: term(), BridgeName :: term()) -> #{status := emqx_resource:resource_status(), error := term()} | {error, Reason :: term()}. health_check(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of + health_check(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +health_check(ConfRootKey, BridgeType, BridgeName) -> + case lookup_conf(ConfRootKey, BridgeType, BridgeName) of #{ enable := true, connector := ConnectorName @@ -608,7 +711,7 @@ health_check(BridgeType, BridgeName) -> connector_type(BridgeType), ConnectorName ), emqx_resource_manager:channel_health_check( - ConnectorId, id(BridgeType, BridgeName, ConnectorName) + ConnectorId, id_with_root_name(ConfRootKey, BridgeType, BridgeName, ConnectorName) ); #{enable := false} -> {error, bridge_stopped}; @@ -617,10 +720,15 @@ health_check(BridgeType, BridgeName) -> end. -spec create_dry_run(bridge_v2_type(), Config :: map()) -> ok | {error, term()}. -create_dry_run(Type, Conf0) -> +create_dry_run(Type, Conf) -> + create_dry_run(?ROOT_KEY_ACTIONS, Type, Conf). + +-spec create_dry_run(root_cfg_key(), bridge_v2_type(), Config :: map()) -> ok | {error, term()}. +create_dry_run(ConfRootKey, Type, Conf0) -> Conf1 = maps:without([<<"name">>], Conf0), TypeBin = bin(Type), - RawConf = #{<<"actions">> => #{TypeBin => #{<<"temp_name">> => Conf1}}}, + ConfRootKeyBin = bin(ConfRootKey), + RawConf = #{ConfRootKeyBin => #{TypeBin => #{<<"temp_name">> => Conf1}}}, %% Check config try _ = @@ -645,6 +753,9 @@ create_dry_run(Type, Conf0) -> end. create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> + create_dry_run_helper(?ROOT_KEY_ACTIONS, BridgeType, ConnectorRawConf, BridgeV2RawConf). + +create_dry_run_helper(ConfRootKey, BridgeType, ConnectorRawConf, BridgeV2RawConf) -> BridgeName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), ConnectorType = connector_type(BridgeType), OnReadyCallback = @@ -652,13 +763,13 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> {_, ConnectorName} = emqx_connector_resource:parse_connector_id(ConnectorId), ChannelTestId = id(BridgeType, BridgeName, ConnectorName), Conf = emqx_utils_maps:unsafe_atom_key_map(BridgeV2RawConf), - ConfWithTypeAndName = Conf#{ - bridge_type => bin(BridgeType), - bridge_name => bin(BridgeName) - }, - case - emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, ConfWithTypeAndName) - of + AugmentedConf = augment_channel_config( + ConfRootKey, + BridgeType, + BridgeName, + Conf + ), + case emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, AugmentedConf) of {error, Reason} -> {error, Reason}; ok -> @@ -677,7 +788,12 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> -spec get_metrics(bridge_v2_type(), bridge_v2_name()) -> emqx_metrics_worker:metrics(). get_metrics(Type, Name) -> - emqx_resource:get_metrics(id(Type, Name)). + get_metrics(?ROOT_KEY_ACTIONS, Type, Name). + +-spec get_metrics(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> + emqx_metrics_worker:metrics(). +get_metrics(ConfRootKey, Type, Name) -> + emqx_resource:get_metrics(id_with_root_name(ConfRootKey, Type, Name)). %%==================================================================== %% On message publish hook (for local topics) @@ -690,7 +806,7 @@ reload_message_publish_hook(Bridges) -> ok = load_message_publish_hook(Bridges). load_message_publish_hook() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), load_message_publish_hook(Bridges). load_message_publish_hook(Bridges) -> @@ -754,7 +870,7 @@ send_to_matched_egress_bridges(Topic, Msg) -> ). get_matched_egress_bridges(Topic) -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), maps:fold( fun(BType, Conf, Acc0) -> maps:fold( @@ -792,24 +908,31 @@ do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) -> parse_id(Id) -> case binary:split(Id, <<":">>, [global]) of [Type, Name] -> - {Type, Name}; + #{kind => undefined, type => Type, name => Name}; [<<"action">>, Type, Name | _] -> - {Type, Name}; + #{kind => action, type => Type, name => Name}; + [<<"source">>, Type, Name | _] -> + #{kind => source, type => Type, name => Name}; _X -> error({error, iolist_to_binary(io_lib:format("Invalid id: ~p", [Id]))}) end. get_channels_for_connector(ConnectorId) -> + Actions = get_channels_for_connector(?ROOT_KEY_ACTIONS, ConnectorId), + Sources = get_channels_for_connector(?ROOT_KEY_SOURCES, ConnectorId), + Actions ++ Sources. + +get_channels_for_connector(SourcesOrActions, ConnectorId) -> try emqx_connector_resource:parse_connector_id(ConnectorId) of {ConnectorType, ConnectorName} -> - RootConf = maps:keys(emqx:get_config([?ROOT_KEY], #{})), + RootConf = maps:keys(emqx:get_config([SourcesOrActions], #{})), RelevantBridgeV2Types = [ Type || Type <- RootConf, connector_type(Type) =:= ConnectorType ], lists:flatten([ - get_channels_for_connector(ConnectorName, BridgeV2Type) + get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) || BridgeV2Type <- RelevantBridgeV2Types ]) catch @@ -819,33 +942,60 @@ get_channels_for_connector(ConnectorId) -> [] end. -get_channels_for_connector(ConnectorName, BridgeV2Type) -> - BridgeV2s = emqx:get_config([?ROOT_KEY, BridgeV2Type], #{}), +get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) -> + BridgeV2s = emqx:get_config([SourcesOrActions, BridgeV2Type], #{}), [ - {id(BridgeV2Type, Name, ConnectorName), Conf#{ - bridge_name => bin(Name), - bridge_type => bin(BridgeV2Type) - }} + { + id_with_root_name(SourcesOrActions, BridgeV2Type, Name, ConnectorName), + augment_channel_config(SourcesOrActions, BridgeV2Type, Name, Conf) + } || {Name, Conf} <- maps:to_list(BridgeV2s), bin(ConnectorName) =:= maps:get(connector, Conf, no_name) ]. %%==================================================================== -%% Exported for tests +%% ID related functions %%==================================================================== id(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of - #{connector := ConnectorName} -> - id(BridgeType, BridgeName, ConnectorName); - {error, Reason} -> - throw(Reason) - end. + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). id(BridgeType, BridgeName, ConnectorName) -> + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, ConnectorName). + +id_with_root_name(RootName, BridgeType, BridgeName) -> + case lookup_conf(RootName, BridgeType, BridgeName) of + #{connector := ConnectorName} -> + id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName); + {error, Reason} -> + throw( + {action_source_not_found, #{ + reason => Reason, + root_name => RootName, + type => BridgeType, + name => BridgeName + }} + ) + end. + +id_with_root_name(RootName0, BridgeType, BridgeName, ConnectorName) -> + RootName = + case bin(RootName0) of + <<"actions">> -> <<"action">>; + <<"sources">> -> <<"source">> + end, ConnectorType = bin(connector_type(BridgeType)), - <<"action:", (bin(BridgeType))/binary, ":", (bin(BridgeName))/binary, ":connector:", - (bin(ConnectorType))/binary, ":", (bin(ConnectorName))/binary>>. + << + (bin(RootName))/binary, + ":", + (bin(BridgeType))/binary, + ":", + (bin(BridgeName))/binary, + ":connector:", + (bin(ConnectorType))/binary, + ":", + (bin(ConnectorName))/binary + >>. connector_type(Type) -> %% remote call so it can be mocked @@ -860,76 +1010,65 @@ bridge_v2_type_to_connector_type(Type) -> import_config(RawConf) -> %% actions structure - emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY, config_key_path()). + emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY_ACTIONS, config_key_path()). %%==================================================================== %% Config Update Handler API %%==================================================================== -config_key_path() -> [?ROOT_KEY]. +config_key_path() -> + [?ROOT_KEY_ACTIONS]. -config_key_path_leaf() -> [?ROOT_KEY, '?', '?']. +config_key_path_leaf() -> + [?ROOT_KEY_ACTIONS, '?', '?']. + +config_key_path_sources() -> + [?ROOT_KEY_SOURCES]. + +config_key_path_leaf_sources() -> + [?ROOT_KEY_SOURCES, '?', '?']. %% enable or disable action -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, undefined) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, undefined) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {error, bridge_not_found}; -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, OldAction) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, OldAction) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {ok, OldAction#{<<"enable">> => operation_to_enable(Oper)}}; %% Updates a single action from a specific HTTP API. %% If the connector is not found, the update operation fails. -pre_config_update([?ROOT_KEY, Type, Name], Conf = #{}, _OldConf) -> - action_convert_from_connector(Type, Name, Conf); +pre_config_update([ConfRootKey, Type, Name], Conf = #{}, _OldConf) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + convert_from_connector(ConfRootKey, Type, Name, Conf); %% Batch updates actions when importing a configuration or executing a CLI command. %% Update succeeded even if the connector is not found, alarm in post_config_update -pre_config_update([?ROOT_KEY], Conf = #{}, _OldConfs) -> - {ok, actions_convert_from_connectors(Conf)}. +pre_config_update([ConfRootKey], Conf = #{}, _OldConfs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + {ok, convert_from_connectors(ConfRootKey, Conf)}. -%% Don't crash event the bridge is not found -post_config_update([?ROOT_KEY, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> - AllBridges = emqx:get_config([?ROOT_KEY]), - case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of - undefined -> - ok; - Action -> - ok = uninstall_bridge_v2(Type, Name, Action), - Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), - reload_message_publish_hook(Bridges) - end, - ?tp(bridge_post_config_update_done, #{}), - ok; -%% Create a single bridge failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) -> - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; -%% update bridges failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) -> - ok = uninstall_bridge_v2(BridgeType, BridgeName, OldConf), - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; %% This top level handler will be triggered when the actions path is updated %% with calls to emqx_conf:update([actions], BridgesConf, #{}). -%% such as import_config/1 -%% Notice ** do succeeded even if the connector is not found ** -%% Install a non-exist connector will alarm & log(warn) in install_bridge_v2. -post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> +post_config_update([ConfRootKey], _Req, NewConf, OldConf, _AppEnv) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), %% The config update will be failed if any task in `perform_bridge_changes` failed. - RemoveFun = fun uninstall_bridge_v2/3, - CreateFun = fun install_bridge_v2/3, + RemoveFun = fun(Type, Name, Conf) -> + uninstall_bridge_v2(ConfRootKey, Type, Name, Conf) + end, + CreateFun = fun(Type, Name, Conf) -> + install_bridge_v2(ConfRootKey, Type, Name, Conf) + end, UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) -> - uninstall_bridge_v2(Type, Name, OldBridgeConf), - install_bridge_v2(Type, Name, Conf) + uninstall_bridge_v2(ConfRootKey, Type, Name, OldBridgeConf), + install_bridge_v2(ConfRootKey, Type, Name, Conf) end, Result = perform_bridge_changes([ #{action => RemoveFun, data => Removed}, @@ -942,7 +1081,45 @@ post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> ]), reload_message_publish_hook(NewConf), ?tp(bridge_post_config_update_done, #{}), - Result. + Result; +%% Don't crash even when the bridge is not found +post_config_update([ConfRootKey, Type, Name], '$remove', _, _OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + AllBridges = emqx:get_config([ConfRootKey]), + case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of + undefined -> + ok; + Action -> + ok = uninstall_bridge_v2(ConfRootKey, Type, Name, Action), + Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), + reload_message_publish_hook(Bridges) + end, + ?tp(bridge_post_config_update_done, #{}), + ok; +%% Create a single bridge fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok; +%% update bridges fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = uninstall_bridge_v2(ConfRootKey, BridgeType, BridgeName, OldConf), + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok. diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( @@ -1026,8 +1203,11 @@ unpack_bridge_conf(Type, PackedConf, TopLevelConf) -> %% * The corresponding bridge v2 should exist %% * The connector for the bridge v2 should have exactly one channel bridge_v1_is_valid(BridgeV1Type, BridgeName) -> + bridge_v1_is_valid(?ROOT_KEY_ACTIONS, BridgeV1Type, BridgeName). + +bridge_v1_is_valid(ConfRootKey, BridgeV1Type, BridgeName) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf(ConfRootKey, BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 true; @@ -1051,20 +1231,45 @@ is_bridge_v2_type(Type) -> emqx_action_info:is_action_type(Type). bridge_v1_list_and_transform() -> - Bridges = list_with_lookup_fun(fun bridge_v1_lookup_and_transform/2), - [B || B <- Bridges, B =/= not_bridge_v1_compatible_error()]. + BridgesFromActions0 = list_with_lookup_fun( + ?ROOT_KEY_ACTIONS, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromActions1 = [ + B + || B <- BridgesFromActions0, + B =/= not_bridge_v1_compatible_error() + ], + FromActionsNames = maps:from_keys([Name || #{name := Name} <- BridgesFromActions1], true), + BridgesFromSources0 = list_with_lookup_fun( + ?ROOT_KEY_SOURCES, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromSources1 = [ + B + || #{name := SourceBridgeName} = B <- BridgesFromSources0, + B =/= not_bridge_v1_compatible_error(), + %% Action is only shown in case of name conflict + not maps:is_key(SourceBridgeName, FromActionsNames) + ], + BridgesFromActions1 ++ BridgesFromSources1. bridge_v1_lookup_and_transform(ActionType, Name) -> - case lookup(ActionType, Name) of - {ok, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> + case lookup_in_actions_or_sources(ActionType, Name) of + {ok, ConfRootKey, + #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), HasBridgeV1Equivalent = has_bridge_v1_equivalent(ActionType), - case HasBridgeV1Equivalent andalso ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of + case + HasBridgeV1Equivalent andalso + ?MODULE:bridge_v1_is_valid(ConfRootKey, BridgeV1Type, Name) + of true -> ConnectorType = connector_type(ActionType), case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( + ConfRootKey, BridgeV1Type, Name, ActionType, @@ -1082,6 +1287,19 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> Error end. +lookup_in_actions_or_sources(ActionType, Name) -> + case lookup(?ROOT_KEY_ACTIONS, ActionType, Name) of + {error, not_found} -> + case lookup(?ROOT_KEY_SOURCES, ActionType, Name) of + {ok, SourceInfo} -> + {ok, ?ROOT_KEY_SOURCES, SourceInfo}; + Error -> + Error + end; + {ok, ActionInfo} -> + {ok, ?ROOT_KEY_ACTIONS, ActionInfo} + end. + not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. @@ -1094,18 +1312,18 @@ has_bridge_v1_equivalent(ActionType) -> connector_raw_config(Connector, ConnectorType) -> get_raw_with_defaults(Connector, ConnectorType, <<"connectors">>, emqx_connector_schema). -action_raw_config(Action, ActionType) -> - get_raw_with_defaults(Action, ActionType, <<"actions">>, emqx_bridge_v2_schema). +action_raw_config(ConfRootName, Action, ActionType) -> + get_raw_with_defaults(Action, ActionType, bin(ConfRootName), emqx_bridge_v2_schema). get_raw_with_defaults(Config, Type, TopLevelConf, SchemaModule) -> RawConfig = maps:get(raw_config, Config), fill_defaults(Type, RawConfig, TopLevelConf, SchemaModule). bridge_v1_lookup_and_transform_helper( - BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector + ConfRootName, BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> ConnectorRawConfig = connector_raw_config(Connector, ConnectorType), - ActionRawConfig = action_raw_config(Action, ActionType), + ActionRawConfig = action_raw_config(ConfRootName, Action, ActionType), BridgeV1Config = emqx_action_info:connector_action_config_to_bridge_v1_config( BridgeV1Type, ConnectorRawConfig, ActionRawConfig ), @@ -1136,7 +1354,52 @@ bridge_v1_lookup_and_transform_helper( end. lookup_conf(Type, Name) -> - case emqx:get_config([?ROOT_KEY, Type, Name], not_found) of + lookup_conf(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(Type, Name) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, Type, Name), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, Type, Name), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + {error, bridge_not_found}; + {{error, bridge_not_found}, Conf} -> + Conf; + {Conf, {error, bridge_not_found}} -> + Conf; + {_Conf1, _Conf2} -> + {error, name_conflict_sources_actions} + end. + +is_only_source(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + false; + {{error, bridge_not_found}, _Conf} -> + true; + {_Conf, {error, bridge_not_found}} -> + false; + {_Conf1, _Conf2} -> + false + end. + +get_conf_root_key_if_only_one(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + error({action_or_source_not_found, BridgeType, BridgeName}); + {{error, bridge_not_found}, _Conf} -> + ?ROOT_KEY_SOURCES; + {_Conf, {error, bridge_not_found}} -> + ?ROOT_KEY_ACTIONS; + {_Conf1, _Conf2} -> + error({name_clash_action_source, BridgeType, BridgeName}) + end. + +lookup_conf(RootName, Type, Name) -> + case emqx:get_config([RootName, Type, Name], not_found) of not_found -> {error, bridge_not_found}; Config -> @@ -1146,7 +1409,7 @@ lookup_conf(Type, Name) -> bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), %% Check if the bridge v2 exists - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, @@ -1158,8 +1421,9 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> true -> %% Using remove + create as update, hence do not delete deps. RemoveDeps = [], + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, BridgeName), PreviousRawConf = emqx:get_raw_config( - [?ROOT_KEY, BridgeV2Type, BridgeName], undefined + [ConfRootKey, BridgeV2Type, BridgeName], undefined ), %% To avoid losing configurations. We have to make sure that no crash occurs %% during deletion and creation of configurations. @@ -1185,17 +1449,18 @@ split_bridge_v1_config_and_create_helper( connector_conf := NewConnectorRawConf, bridge_v2_type := BridgeType, bridge_v2_name := BridgeName, - bridge_v2_conf := NewBridgeV2RawConf + bridge_v2_conf := NewBridgeV2RawConf, + conf_root_key := ConfRootName } = split_and_validate_bridge_v1_config( BridgeV1Type, BridgeName, RawConf, PreviousRawConf ), - _ = PreCreateFun(), do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1210,6 +1475,7 @@ split_bridge_v1_config_and_create_helper( end. do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1220,7 +1486,7 @@ do_connector_and_bridge_create( ) -> case emqx_connector:create(ConnectorType, NewConnectorName, NewConnectorRawConf) of {ok, _} -> - case create(BridgeType, BridgeName, NewBridgeV2RawConf) of + case create(ConfRootName, BridgeType, BridgeName, NewBridgeV2RawConf) of {ok, _} = Result -> Result; {error, Reason1} -> @@ -1257,10 +1523,15 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR } } }, + ConfRootKeyPrevRawConf = + case PreviousRawConf =/= undefined of + true -> get_conf_root_key_if_only_one(BridgeV2Type, BridgeName); + false -> not_used + end, FakeGlobalConfig = emqx_utils_maps:put_if( FakeGlobalConfig0, - bin(?ROOT_KEY), + bin(ConfRootKeyPrevRawConf), #{bin(BridgeV2Type) => #{bin(BridgeName) => PreviousRawConf}}, PreviousRawConf =/= undefined ), @@ -1269,10 +1540,11 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR Output = emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2( FakeGlobalConfig ), + ConfRootKey = get_conf_root_key(Output), NewBridgeV2RawConf = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName) ], @@ -1280,7 +1552,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR ), ConnectorName = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName), <<"connector">> @@ -1303,7 +1575,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR bin(ConnectorName) => NewConnectorRawConf } }, - <<"actions">> => #{ + ConfRootKey => #{ bin(BridgeV2Type) => #{ bin(BridgeName) => NewBridgeV2RawConf } @@ -1323,7 +1595,8 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR connector_conf => NewConnectorRawConf, bridge_v2_type => BridgeV2Type, bridge_v2_name => BridgeName, - bridge_v2_conf => NewBridgeV2RawConf + bridge_v2_conf => NewBridgeV2RawConf, + conf_root_key => ConfRootKey } catch %% validation errors @@ -1331,6 +1604,13 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR {error, Reason1} end. +get_conf_root_key(#{<<"actions">> := _}) -> + <<"actions">>; +get_conf_root_key(#{<<"sources">> := _}) -> + <<"sources">>; +get_conf_root_key(_NoMatch) -> + error({incompatible_bridge_v1, no_action_or_source}). + bridge_v1_create_dry_run(BridgeType, RawConfig0) -> RawConf = maps:without([<<"name">>], RawConfig0), TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), @@ -1356,7 +1636,7 @@ bridge_v1_remove(BridgeV1Type, BridgeName) -> bridge_v1_remove( ActionType, BridgeName, - lookup_conf(ActionType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(ActionType, BridgeName) ). bridge_v1_remove( @@ -1364,7 +1644,8 @@ bridge_v1_remove( Name, #{connector := ConnectorName} ) -> - case remove(ActionType, Name) of + ConfRootKey = get_conf_root_key_if_only_one(ActionType, Name), + case remove(ConfRootKey, ActionType, Name) of ok -> ConnectorType = connector_type(ActionType), emqx_connector:remove(ConnectorType, ConnectorName); @@ -1384,7 +1665,7 @@ bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) -> BridgeV2Type, BridgeName, RemoveDeps, - lookup_conf(BridgeV2Type, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) ). %% Bridge v1 delegated-removal in 3 steps: @@ -1398,9 +1679,10 @@ bridge_v1_check_deps_and_remove( #{connector := ConnectorName} ) -> RemoveConnector = lists:member(connector, RemoveDeps), - case emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of + case maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of ok -> - case remove(BridgeType, BridgeName) of + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + case remove(ConfRootKey, BridgeType, BridgeName) of ok when RemoveConnector -> maybe_delete_channels(BridgeType, BridgeName, ConnectorName); ok -> @@ -1415,6 +1697,14 @@ bridge_v1_check_deps_and_remove(_BridgeType, _BridgeName, _RemoveDeps, Error) -> %% TODO: the connector is gone, for whatever reason, maybe call remove/2 anyway? Error. +maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) -> + case is_only_source(BridgeType, BridgeName) of + true -> + ok; + false -> + emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) + end. + maybe_delete_channels(BridgeType, BridgeName, ConnectorName) -> case connector_has_channels(BridgeType, ConnectorName) of true -> @@ -1446,11 +1736,14 @@ connector_has_channels(BridgeV2Type, ConnectorName) -> end. bridge_v1_id_to_connector_resource_id(BridgeId) -> + bridge_v1_id_to_connector_resource_id(?ROOT_KEY_ACTIONS, BridgeId). + +bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, Name] -> BridgeV2Type = bin(bridge_v1_type_to_bridge_v2_type(Type)), ConnectorName = - case lookup_conf(BridgeV2Type, Name) of + case lookup_conf(ConfRootKey, BridgeV2Type, Name) of #{connector := Con} -> Con; {error, Reason} -> @@ -1467,23 +1760,25 @@ bridge_v1_enable_disable(Action, BridgeType, BridgeName) -> Action, BridgeType, BridgeName, - lookup_conf(BridgeType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeType, BridgeName) ); false -> {error, not_bridge_v1_compatible} end. -bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, bridge_not_found}) -> - {error, bridge_not_found}; +bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, Reason}) -> + {error, Reason}; bridge_v1_enable_disable_helper(enable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), {ok, _} = emqx_connector:disable_enable(enable, ConnectorType, ConnectorName), - emqx_bridge_v2:disable_enable(enable, BridgeV2Type, BridgeName); + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_bridge_v2:disable_enable(ConfRootKey, enable, BridgeV2Type, BridgeName); bridge_v1_enable_disable_helper(disable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), - {ok, _} = emqx_bridge_v2:disable_enable(disable, BridgeV2Type, BridgeName), + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + {ok, _} = emqx_bridge_v2:disable_enable(ConfRootKey, disable, BridgeV2Type, BridgeName), emqx_connector:disable_enable(disable, ConnectorType, ConnectorName). bridge_v1_restart(BridgeV1Type, Name) -> @@ -1508,10 +1803,12 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), case emqx_bridge_v2:bridge_v1_is_valid(BridgeV1Type, Name) of true -> + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, Name), connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ); @@ -1519,6 +1816,13 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> {error, not_bridge_v1_compatible} end. +bridge_v1_reset_metrics(BridgeV1Type, BridgeName) -> + BridgeV2Type = bridge_v1_type_to_bridge_v2_type(BridgeV1Type), + ConfRootKey = get_conf_root_key_if_only_one( + BridgeV2Type, BridgeName + ), + ok = reset_metrics(ConfRootKey, BridgeV2Type, BridgeName). + %%==================================================================== %% Misc helper functions %%==================================================================== @@ -1559,12 +1863,12 @@ referenced_connectors_exist(BridgeType, ConnectorNameBin, BridgeName) -> ok end. -actions_convert_from_connectors(Conf) -> +convert_from_connectors(ConfRootKey, Conf) -> maps:map( fun(ActionType, Actions) -> maps:map( fun(ActionName, Action) -> - case action_convert_from_connector(ActionType, ActionName, Action) of + case convert_from_connector(ConfRootKey, ActionType, ActionName, Action) of {ok, NewAction} -> NewAction; {error, _} -> Action end @@ -1575,7 +1879,7 @@ actions_convert_from_connectors(Conf) -> Conf ). -action_convert_from_connector(Type, Name, Action = #{<<"connector">> := ConnectorName}) -> +convert_from_connector(ConfRootKey, Type, Name, Action = #{<<"connector">> := ConnectorName}) -> case get_connector_info(ConnectorName, Type) of {ok, Connector} -> Action1 = emqx_action_info:action_convert_from_connector(Type, Connector, Action), @@ -1585,7 +1889,8 @@ action_convert_from_connector(Type, Name, Action = #{<<"connector">> := Connecto bridge_name => Name, reason => <<"connector_not_found_or_wrong_type">>, bridge_type => Type, - connector_name => ConnectorName + connector_name => ConnectorName, + conf_root_key => ConfRootKey }} end. diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 254390a36..e8a500e85 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -26,6 +26,9 @@ -import(hoconsc, [mk/2, array/1, enum/1]). -import(emqx_utils, [redact/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% Swagger specs from hocon schema -export([ api_spec/0, @@ -34,7 +37,7 @@ namespace/0 ]). -%% API callbacks +%% API callbacks : actions -export([ '/actions'/2, '/actions/:id'/2, @@ -46,9 +49,28 @@ '/actions_probe'/2, '/action_types'/2 ]). +%% API callbacks : sources +-export([ + '/sources'/2, + '/sources/:id'/2, + '/sources/:id/metrics'/2, + '/sources/:id/metrics/reset'/2, + '/sources/:id/enable/:enable'/2, + '/sources/:id/:operation'/2, + '/nodes/:node/sources/:id/:operation'/2, + '/sources_probe'/2, + '/source_types'/2 +]). %% BpAPI / RPC Targets --export([lookup_from_local_node/2, get_metrics_from_local_node/2]). +-export([ + lookup_from_local_node/2, + get_metrics_from_local_node/2, + lookup_from_local_node_v6/3, + get_metrics_from_local_node_v6/3 +]). + +-define(BPAPI_NAME, emqx_bridge). -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( @@ -71,13 +93,16 @@ end ). -namespace() -> "actions". +namespace() -> "actions_and_sources". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ + %%============= + %% Actions + %%============= "/actions", "/actions/:id", "/actions/:id/enable/:enable", @@ -88,7 +113,21 @@ paths() -> "/actions/:id/metrics", "/actions/:id/metrics/reset", "/actions_probe", - "/action_types" + "/action_types", + %%============= + %% Sources + %%============= + "/sources", + "/sources/:id", + "/sources/:id/enable/:enable", + "/sources/:id/:operation", + "/nodes/:node/sources/:id/:operation", + %% %% Caveat: metrics paths must come *after* `/:operation', otherwise minirest will + %% %% try to match the latter first, trying to interpret `metrics' as an operation... + "/sources/:id/metrics", + "/sources/:id/metrics/reset", + "/sources_probe" + %% "/source_types" ]. error_schema(Code, Message) -> @@ -101,17 +140,28 @@ error_schema(Codes, Message, ExtraFields) when is_list(Message) -> error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) -> ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message). -get_response_body_schema() -> +actions_get_response_body_schema() -> emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:get_response(), - bridge_info_examples(get) + emqx_bridge_v2_schema:actions_get_response(), + bridge_info_examples(get, ?ROOT_KEY_ACTIONS) ). -bridge_info_examples(Method) -> - emqx_bridge_v2_schema:examples(Method). +sources_get_response_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_get_response(), + bridge_info_examples(get, ?ROOT_KEY_SOURCES) + ). -bridge_info_array_example(Method) -> - lists:map(fun(#{value := Config}) -> Config end, maps:values(bridge_info_examples(Method))). +bridge_info_examples(Method, ?ROOT_KEY_ACTIONS) -> + emqx_bridge_v2_schema:actions_examples(Method); +bridge_info_examples(Method, ?ROOT_KEY_SOURCES) -> + emqx_bridge_v2_schema:sources_examples(Method). + +bridge_info_array_example(Method, ConfRootKey) -> + lists:map( + fun(#{value := Config}) -> Config end, + maps:values(bridge_info_examples(Method, ConfRootKey)) + ). param_path_id() -> {id, @@ -185,6 +235,9 @@ param_path_enable() -> } )}. +%%================================================================================ +%% Actions +%%================================================================================ schema("/actions") -> #{ 'operationId' => '/actions', @@ -194,8 +247,8 @@ schema("/actions") -> description => ?DESC("desc_api1"), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - array(emqx_bridge_v2_schema:get_response()), - bridge_info_array_example(get) + array(emqx_bridge_v2_schema:actions_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_ACTIONS) ) } }, @@ -204,11 +257,11 @@ schema("/actions") -> summary => <<"Create bridge">>, description => ?DESC("desc_api2"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ - 201 => get_response_body_schema(), + 201 => actions_get_response_body_schema(), 400 => error_schema('ALREADY_EXISTS', "Bridge already exists") } } @@ -222,7 +275,7 @@ schema("/actions/:id") -> description => ?DESC("desc_api3"), parameters => [param_path_id()], responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found") } }, @@ -232,11 +285,11 @@ schema("/actions/:id") -> description => ?DESC("desc_api4"), parameters => [param_path_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:put_request(), - bridge_info_examples(put) + emqx_bridge_v2_schema:actions_put_request(), + bridge_info_examples(put, ?ROOT_KEY_ACTIONS) ), responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found"), 400 => error_schema('BAD_REQUEST', "Update bridge failed") } @@ -361,8 +414,8 @@ schema("/actions_probe") -> desc => ?DESC("desc_api9"), summary => <<"Test creating bridge">>, 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ 204 => <<"Test bridge OK">>, @@ -379,12 +432,223 @@ schema("/action_types") -> summary => <<"List available action types">>, responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - array(emqx_bridge_v2_schema:types_sc()), + array(emqx_bridge_v2_schema:action_types_sc()), #{ <<"types">> => #{ summary => <<"Action types">>, - value => emqx_bridge_v2_schema:types() + value => emqx_bridge_v2_schema:action_types() + } + } + ) + } + } + }; +%%================================================================================ +%% Sources +%%================================================================================ +schema("/sources") -> + #{ + 'operationId' => '/sources', + get => #{ + tags => [<<"sources">>], + summary => <<"List sources">>, + description => ?DESC("desc_api1"), + responses => #{ + %% FIXME: examples + 200 => emqx_dashboard_swagger:schema_with_example( + array(emqx_bridge_v2_schema:sources_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_SOURCES) + ) + } + }, + post => #{ + tags => [<<"sources">>], + summary => <<"Create source">>, + description => ?DESC("desc_api2"), + %% FIXME: examples + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 201 => sources_get_response_body_schema(), + 400 => error_schema('ALREADY_EXISTS', "Source already exists") + } + } + }; +schema("/sources/:id") -> + #{ + 'operationId' => '/sources/:id', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source">>, + description => ?DESC("desc_api3"), + parameters => [param_path_id()], + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + }, + put => #{ + tags => [<<"sources">>], + summary => <<"Update source">>, + description => ?DESC("desc_api4"), + parameters => [param_path_id()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_put_request(), + bridge_info_examples(put, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found"), + 400 => error_schema('BAD_REQUEST', "Update source failed") + } + }, + delete => #{ + tags => [<<"sources">>], + summary => <<"Delete source">>, + description => ?DESC("desc_api5"), + parameters => [param_path_id(), param_qs_delete_cascade()], + responses => #{ + 204 => <<"Source deleted">>, + 400 => error_schema( + 'BAD_REQUEST', + "Cannot delete bridge while active rules are defined for this source", + [{rules, mk(array(string()), #{desc => "Dependent Rule IDs"})}] + ), + 404 => error_schema('NOT_FOUND', "Source not found"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/metrics") -> + #{ + 'operationId' => '/sources/:id/metrics', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source metrics">>, + description => ?DESC("desc_bridge_metrics"), + parameters => [param_path_id()], + responses => #{ + 200 => emqx_bridge_schema:metrics_fields(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/metrics/reset") -> + #{ + 'operationId' => '/sources/:id/metrics/reset', + put => #{ + tags => [<<"sources">>], + summary => <<"Reset source metrics">>, + description => ?DESC("desc_api6"), + parameters => [param_path_id()], + responses => #{ + 204 => <<"Reset success">>, + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/enable/:enable") -> + #{ + 'operationId' => '/sources/:id/enable/:enable', + put => + #{ + tags => [<<"sources">>], + summary => <<"Enable or disable bridge">>, + desc => ?DESC("desc_enable_bridge"), + parameters => [param_path_id(), param_path_enable()], + responses => + #{ + 204 => <<"Success">>, + 404 => error_schema( + 'NOT_FOUND', "Bridge not found or invalid operation" + ), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/:operation") -> + #{ + 'operationId' => '/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge">>, + description => ?DESC("desc_api7"), + parameters => [ + param_path_id(), + param_path_operation_cluster() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', "Problem with configuration of external service" + ), + 404 => error_schema('NOT_FOUND', "Bridge not found or invalid operation"), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/nodes/:node/sources/:id/:operation") -> + #{ + 'operationId' => '/nodes/:node/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge on a given node">>, + description => ?DESC("desc_api8"), + parameters => [ + param_path_node(), + param_path_id(), + param_path_operation_on_node() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', + "Problem with configuration of external service or bridge not enabled" + ), + 404 => error_schema( + 'NOT_FOUND', "Bridge or node not found or invalid operation" + ), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources_probe") -> + #{ + 'operationId' => '/sources_probe', + post => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api9"), + summary => <<"Test creating bridge">>, + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 204 => <<"Test bridge OK">>, + 400 => error_schema(['TEST_FAILED'], "bridge test failed") + } + } + }; +schema("/source_types") -> + #{ + 'operationId' => '/source_types', + get => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api10"), + summary => <<"List available source types">>, + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + array(emqx_bridge_v2_schema:action_types_sc()), + #{ + <<"types">> => + #{ + summary => <<"Source types">>, + value => emqx_bridge_v2_schema:action_types() } } ) @@ -392,21 +656,103 @@ schema("/action_types") -> } }. +%%------------------------------------------------------------------------------ +%% Thin Handlers +%%------------------------------------------------------------------------------ +%%================================================================================ +%% Actions +%%================================================================================ '/actions'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, _} -> - ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); - {error, not_found} -> - Conf = filter_out_request_body(Conf0), - create_bridge(BridgeType, BridgeName, Conf) - end; + handle_create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, Conf0); '/actions'(get, _Params) -> - Nodes = mria:running_nodes(), - NodeReplies = emqx_bridge_proto_v5:v2_list_bridges_on_nodes(Nodes), + handle_list(?ROOT_KEY_ACTIONS). + +'/actions/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, 200)); +'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_ACTIONS, Id, Conf0); +'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_ACTIONS, Id, Qs). + +'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName)). + +'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_ACTIONS, Id). + +'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_ACTIONS, Id, Enable). + +'/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_ACTIONS, Id, Op). + +'/nodes/:node/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_ACTIONS, Node, Id, Op). + +'/actions_probe'(post, Request) -> + handle_probe(?ROOT_KEY_ACTIONS, Request). + +'/action_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:action_types()). +%%================================================================================ +%% Sources +%%================================================================================ +'/sources'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> + handle_create(?ROOT_KEY_SOURCES, BridgeType, BridgeName, Conf0); +'/sources'(get, _Params) -> + handle_list(?ROOT_KEY_SOURCES). + +'/sources/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName, 200)); +'/sources/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_SOURCES, Id, Conf0); +'/sources/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_SOURCES, Id, Qs). + +'/sources/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName)). + +'/sources/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_SOURCES, Id). + +'/sources/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_SOURCES, Id, Enable). + +'/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_SOURCES, Id, Op). + +'/nodes/:node/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_SOURCES, Node, Id, Op). + +'/sources_probe'(post, Request) -> + handle_probe(?ROOT_KEY_SOURCES, Request). + +'/source_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:source_types()). + +%%------------------------------------------------------------------------------ +%% Handlers +%%------------------------------------------------------------------------------ + +handle_list(ConfRootKey) -> + Nodes = emqx:running_nodes(), + NodeReplies = emqx_bridge_proto_v6:v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey), case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ - [format_resource(Data, Node) || Data <- Bridges] + [format_resource(ConfRootKey, Data, Node) || Data <- Bridges] || {Node, Bridges} <- lists:zip(Nodes, NodeBridges) ], ?OK(zip_bridges(AllBridges)); @@ -414,34 +760,44 @@ schema("/action_types") -> ?INTERNAL_ERROR(Reason) end. -'/actions/:id'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); -'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> +handle_create(ConfRootKey, Type, Name, Conf0) -> + case emqx_bridge_v2:lookup(ConfRootKey, Type, Name) of + {ok, _} -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); + {error, not_found} -> + Conf = filter_out_request_body(Conf0), + create_bridge(ConfRootKey, Type, Name, Conf) + end. + +handle_update(ConfRootKey, Id, Conf0) -> Conf1 = filter_out_request_body(Conf0), ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), Conf = emqx_utils:deobfuscate(Conf1, RawConf), - update_bridge(BridgeType, BridgeName, Conf); + update_bridge(ConfRootKey, BridgeType, BridgeName, Conf); {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end - ); -'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + ). + +handle_delete(ConfRootKey, Id, QueryStringOpts) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> AlsoDeleteActions = - case maps:get(<<"also_delete_dep_actions">>, Qs, <<"false">>) of + case maps:get(<<"also_delete_dep_actions">>, QueryStringOpts, <<"false">>) of <<"true">> -> true; true -> true; _ -> false end, case - emqx_bridge_v2:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) + emqx_bridge_v2:check_deps_and_remove( + ConfRootKey, BridgeType, BridgeName, AlsoDeleteActions + ) of ok -> ?NO_CONTENT; @@ -465,23 +821,22 @@ schema("/action_types") -> end ). -'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). - -'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> +handle_reset_metrics(ConfRootKey, Id) -> ?TRY_PARSE_ID( Id, begin ActionType = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(ActionType, BridgeName), + ok = emqx_bridge_v2:reset_metrics(ConfRootKey, ActionType, BridgeName), ?NO_CONTENT end ). -'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> +handle_disable_enable(ConfRootKey, Id, Enable) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:disable_enable(enable_func(Enable), BridgeType, BridgeName) of + case + emqx_bridge_v2:disable_enable(ConfRootKey, enable_func(Enable), BridgeType, BridgeName) + of {ok, _} -> ?NO_CONTENT; {error, {pre_config_update, _, bridge_not_found}} -> @@ -495,41 +850,42 @@ schema("/action_types") -> end ). -'/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op} -}) -> +handle_operation(ConfRootKey, Id, Op) -> ?TRY_PARSE_ID( Id, begin OperFunc = operation_func(all, Op), - Nodes = mria:running_nodes(), - call_operation_if_enabled(all, OperFunc, [Nodes, BridgeType, BridgeName]) + Nodes = emqx:running_nodes(), + call_operation_if_enabled(all, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) end ). -'/nodes/:node/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op, node := Node} -}) -> +handle_node_operation(ConfRootKey, Node, Id, Op) -> ?TRY_PARSE_ID( Id, case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> OperFunc = operation_func(TargetNode, Op), - call_operation_if_enabled(TargetNode, OperFunc, [TargetNode, BridgeType, BridgeName]); + call_operation_if_enabled(TargetNode, OperFunc, [ + TargetNode, ConfRootKey, BridgeType, BridgeName + ]); {error, _} -> ?NOT_FOUND(<<"Invalid node name: ", Node/binary>>) end ). -'/actions_probe'(post, Request) -> - RequestMeta = #{module => ?MODULE, method => post, path => "/actions_probe"}, +handle_probe(ConfRootKey, Request) -> + Path = + case ConfRootKey of + ?ROOT_KEY_ACTIONS -> "/actions_probe"; + ?ROOT_KEY_SOURCES -> "/sources_probe" + end, + RequestMeta = #{module => ?MODULE, method => post, path => Path}, case emqx_dashboard_swagger:filter_check_request_and_translate_body(Request, RequestMeta) of - {ok, #{body := #{<<"type">> := ConnType} = Params}} -> + {ok, #{body := #{<<"type">> := Type} = Params}} -> Params1 = maybe_deobfuscate_bridge_probe(Params), Params2 = maps:remove(<<"type">>, Params1), - case emqx_bridge_v2:create_dry_run(ConnType, Params2) of + case emqx_bridge_v2:create_dry_run(ConfRootKey, Type, Params2) of ok -> ?NO_CONTENT; {error, #{kind := validation_error} = Reason0} -> @@ -548,9 +904,7 @@ schema("/action_types") -> redact(BadRequest) end. -'/action_types'(get, _Request) -> - ?OK(emqx_bridge_v2_schema:types()). - +%%% API helpers maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) -> case emqx_bridge_v2:lookup(ActionType, BridgeName) of {ok, #{raw_config := RawConf}} -> @@ -564,7 +918,6 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeN maybe_deobfuscate_bridge_probe(Params) -> Params. -%%% API helpers is_ok(ok) -> ok; is_ok(OkResult = {ok, _}) -> @@ -587,9 +940,16 @@ is_ok(ResL) -> end. %% bridge helpers -lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> - Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v5:v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of +-spec lookup_from_all_nodes(emqx_bridge_v2:root_cfg_key(), _, _, _) -> _. +lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, SuccCode) -> + Nodes = emqx:running_nodes(), + case + is_ok( + emqx_bridge_proto_v6:v2_lookup_from_all_nodes_v6( + Nodes, ConfRootKey, BridgeType, BridgeName + ) + ) + of {ok, [{ok, _} | _] = Results} -> {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> @@ -598,10 +958,10 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> ?INTERNAL_ERROR(Reason) end. -get_metrics_from_all_nodes(ActionType, ActionName) -> +get_metrics_from_all_nodes(ConfRootKey, Type, Name) -> Nodes = emqx:running_nodes(), Result = maybe_unwrap( - emqx_bridge_proto_v5:v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) + emqx_bridge_proto_v6:v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, Type, Name) ), case Result of Metrics when is_list(Metrics) -> @@ -610,22 +970,25 @@ get_metrics_from_all_nodes(ActionType, ActionName) -> ?INTERNAL_ERROR(Reason) end. -operation_func(all, start) -> v2_start_bridge_to_all_nodes; -operation_func(_Node, start) -> v2_start_bridge_to_node. +operation_func(all, start) -> v2_start_bridge_on_all_nodes_v6; +operation_func(_Node, start) -> v2_start_bridge_on_node_v6; +operation_func(all, lookup) -> v2_lookup_from_all_nodes_v6; +operation_func(all, list) -> v2_list_bridges_on_nodes_v6; +operation_func(all, get_metrics) -> v2_get_metrics_from_all_nodes_v6. -call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) -> - try is_enabled_bridge(BridgeType, BridgeName) of +call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) -> + try is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) of false -> ?BRIDGE_NOT_ENABLED; true -> - call_operation(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) + call_operation(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) catch throw:not_found -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end. -is_enabled_bridge(BridgeType, BridgeName) -> - try emqx_bridge_v2:lookup(BridgeType, binary_to_existing_atom(BridgeName)) of +is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) -> + try emqx_bridge_v2:lookup(ConfRootKey, BridgeType, binary_to_existing_atom(BridgeName)) of {ok, #{raw_config := ConfMap}} -> maps:get(<<"enable">>, ConfMap, false); {error, not_found} -> @@ -637,7 +1000,7 @@ is_enabled_bridge(BridgeType, BridgeName) -> throw(not_found) end. -call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> +call_operation(NodeOrAll, OperFunc, Args = [_Nodes, _ConfRootKey, BridgeType, BridgeName]) -> case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok -> ?NO_CONTENT; @@ -668,12 +1031,12 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> - case lists:member(Node, mria:running_nodes()) of + case lists:member(Node, emqx:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. @@ -681,7 +1044,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(Version, Call, Args) -> case is_supported_version(Version, Call) of true -> - apply(emqx_bridge_proto_v5, Call, Args); + apply(emqx_bridge_proto_v6, Call, Args); false -> {error, not_implemented} end. @@ -689,7 +1052,12 @@ do_bpapi_call_vsn(Version, Call, Args) -> is_supported_version(Version, Call) -> lists:member(Version, supported_versions(Call)). -supported_versions(_Call) -> [5]. +supported_versions(_Call) -> bpapi_version_range(6, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). maybe_unwrap({error, not_implemented}) -> {error, not_implemented}; @@ -763,7 +1131,15 @@ aggregate_status(AllStatus) -> %% RPC Target lookup_from_local_node(BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resource(Res, node())}; + {ok, Res} -> {ok, format_resource(?ROOT_KEY_ACTIONS, Res, node())}; + Error -> Error + end. + +%% RPC Target +-spec lookup_from_local_node_v6(emqx_bridge_v2:root_cfg_key(), _, _) -> _. +lookup_from_local_node_v6(ConfRootKey, BridgeType, BridgeName) -> + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of + {ok, Res} -> {ok, format_resource(ConfRootKey, Res, node())}; Error -> Error end. @@ -771,8 +1147,13 @@ lookup_from_local_node(BridgeType, BridgeName) -> get_metrics_from_local_node(ActionType, ActionName) -> format_metrics(emqx_bridge_v2:get_metrics(ActionType, ActionName)). +%% RPC Target +get_metrics_from_local_node_v6(ConfRootKey, Type, Name) -> + format_metrics(emqx_bridge_v2:get_metrics(ConfRootKey, Type, Name)). + %% resource format_resource( + ConfRootKey, #{ type := Type, name := Name, @@ -783,7 +1164,7 @@ format_resource( }, Node ) -> - RawConf = fill_defaults(Type, RawConf0), + RawConf = fill_defaults(ConfRootKey, Type, RawConf0), redact( maps:merge( RawConf#{ @@ -914,17 +1295,18 @@ aggregate_metrics( M17 + N17 ). -fill_defaults(Type, RawConf) -> - PackedConf = pack_bridge_conf(Type, RawConf), +fill_defaults(ConfRootKey, Type, RawConf) -> + PackedConf = pack_bridge_conf(ConfRootKey, Type, RawConf), FullConf = emqx_config:fill_defaults(emqx_bridge_v2_schema, PackedConf, #{}), - unpack_bridge_conf(Type, FullConf). + unpack_bridge_conf(ConfRootKey, Type, FullConf). -pack_bridge_conf(Type, RawConf) -> - #{<<"actions">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. +pack_bridge_conf(ConfRootKey, Type, RawConf) -> + #{bin(ConfRootKey) => #{bin(Type) => #{<<"foo">> => RawConf}}}. -unpack_bridge_conf(Type, PackedConf) -> +unpack_bridge_conf(ConfRootKey, Type, PackedConf) -> + ConfRootKeyBin = bin(ConfRootKey), TypeBin = bin(Type), - #{<<"actions">> := Bridges} = PackedConf, + #{ConfRootKeyBin := Bridges} = PackedConf, #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), RawConf. @@ -938,13 +1320,13 @@ format_resource_data(error, Error, Result) -> format_resource_data(K, V, Result) -> Result#{K => V}. -create_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 201). +create_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 201). -update_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 200). +update_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 200). -create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> +create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> Check = try is_binary(BridgeType) andalso emqx_resource:validate_type(BridgeType), @@ -955,15 +1337,15 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> end, case Check of ok -> - do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode); + do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode); BadRequest -> BadRequest end. -do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> - case emqx_bridge_v2:create(BridgeType, BridgeName, Conf) of +do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> + case emqx_bridge_v2:create(ConfRootKey, BridgeType, BridgeName, Conf) of {ok, _} -> - lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode); + lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, HttpStatusCode); {error, {PreOrPostConfigUpdate, _HandlerMod, Reason}} when PreOrPostConfigUpdate =:= pre_config_update; PreOrPostConfigUpdate =:= post_config_update diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl new file mode 100644 index 000000000..fbcef8b5c --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v6). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + get_metrics_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3, + + %% introduced in v6 + v2_lookup_from_all_nodes_v6/4, + v2_list_bridges_on_nodes_v6/2, + v2_get_metrics_from_all_nodes_v6/4, + v2_start_bridge_on_node_v6/4, + v2_start_bridge_on_all_nodes_v6/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.5.0". + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(term()). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()). +get_metrics_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + get_metrics_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +%%-------------------------------------------------------------------------------- +%% introduced in v6 +%%-------------------------------------------------------------------------------- + +%% V2 Calls +-spec v2_lookup_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_lookup_from_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + lookup_from_local_node_v6, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_list_bridges_on_nodes_v6([node()], emqx_bridge_v2:root_cfg_key()) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey) -> + erpc:multicall(Nodes, emqx_bridge_v2, list, [ConfRootKey], ?TIMEOUT). + +-spec v2_get_metrics_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, ActionType, ActionName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + get_metrics_from_local_node_v6, + [ConfRootKey, ActionType, ActionName], + ?TIMEOUT + ). + +-spec v2_start_bridge_on_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(ok). +v2_start_bridge_on_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_start_bridge_on_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> + term(). +v2_start_bridge_on_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 514eb6988..2708df87d 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -28,22 +28,33 @@ -export([roots/0, fields/1, desc/1, namespace/0, tags/0]). -export([ - get_response/0, - put_request/0, - post_request/0, - examples/1, + actions_get_response/0, + actions_put_request/0, + actions_post_request/0, + actions_examples/1, action_values/4 ]). +-export([ + sources_get_response/0, + sources_put_request/0, + sources_post_request/0, + sources_examples/1, + source_values/4 +]). + %% Exported for mocking %% TODO: refactor emqx_bridge_v1_compatibility_layer_SUITE so we don't need to %% export this -export([ - registered_api_schemas/1 + registered_actions_api_schemas/1, + registered_sources_api_schemas/1 ]). --export([types/0, types_sc/0]). --export([resource_opts_fields/0, resource_opts_fields/1]). +-export([action_types/0, action_types_sc/0]). +-export([source_types/0, source_types_sc/0]). +-export([action_resource_opts_fields/0, action_resource_opts_fields/1]). +-export([source_resource_opts_fields/0, source_resource_opts_fields/1]). -export([ api_fields/3 @@ -53,38 +64,146 @@ make_producer_action_schema/1, make_producer_action_schema/2, make_consumer_action_schema/1, make_consumer_action_schema/2, top_level_common_action_keys/0, - project_to_actions_resource_opts/1 + project_to_actions_resource_opts/1, + project_to_sources_resource_opts/1 ]). -export([actions_convert_from_connectors/1]). --export_type([action_type/0]). +-export_type([action_type/0, source_type/0]). %% Should we explicitly list them here so dialyzer may be more helpful? -type action_type() :: atom(). +-type source_type() :: atom(). +-type http_method() :: get | post | put. +-type schema_example_map() :: #{atom() => term()}. %%====================================================================================== %% For HTTP APIs -get_response() -> - api_schema("get"). +%%====================================================================================== -put_request() -> - api_schema("put"). +%%--------------------------------------------- +%% Actions +%%--------------------------------------------- -post_request() -> - api_schema("post"). +actions_get_response() -> + actions_api_schema("get"). -api_schema(Method) -> - APISchemas = ?MODULE:registered_api_schemas(Method), +actions_put_request() -> + actions_api_schema("put"). + +actions_post_request() -> + actions_api_schema("post"). + +actions_api_schema(Method) -> + APISchemas = ?MODULE:registered_actions_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). -registered_api_schemas(Method) -> - RegisteredSchemas = emqx_action_info:registered_schema_modules(), +registered_actions_api_schemas(Method) -> + RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_bridge_v2") || {BridgeV2Type, SchemaModule} <- RegisteredSchemas ]. +-spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +action_values(Method, ActionType, ConnectorType, ActionValues) -> + ActionTypeBin = atom_to_binary(ActionType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", ActionTypeBin/binary, " action">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + ActionValues, + method_values(action, Method, ActionType) + ] + ). + +actions_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Sources +%%--------------------------------------------- + +sources_get_response() -> + sources_api_schema("get"). + +sources_put_request() -> + sources_api_schema("put"). + +sources_post_request() -> + sources_api_schema("post"). + +sources_api_schema(Method) -> + APISchemas = ?MODULE:registered_sources_api_schemas(Method), + hoconsc:union(bridge_api_union(APISchemas)). + +registered_sources_api_schemas(Method) -> + RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(), + [ + api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_source") + || {BridgeV2Type, SchemaModule} <- RegisteredSchemas + ]. + +-spec source_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +source_values(Method, SourceType, ConnectorType, SourceValues) -> + SourceTypeBin = atom_to_binary(SourceType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", SourceTypeBin/binary, " source">>, + connector => <>, + resource_opts => #{ + health_check_interval => <<"30s">> + } + }, + [ + SourceValues, + method_values(source, Method, SourceType) + ] + ). + +sources_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, source_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_sources()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Common helpers +%%--------------------------------------------- + api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. @@ -111,41 +230,17 @@ bridge_api_union(Refs) -> end end. --type http_method() :: get | post | put. --type schema_example_map() :: #{atom() => term()}. - --spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). -action_values(Method, ActionType, ConnectorType, ActionValues) -> - ActionTypeBin = atom_to_binary(ActionType), - ConnectorTypeBin = atom_to_binary(ConnectorType), - lists:foldl( - fun(M1, M2) -> - maps:merge(M1, M2) - end, - #{ - enable => true, - description => <<"My example ", ActionTypeBin/binary, " action">>, - connector => <>, - resource_opts => #{ - health_check_interval => "30s" - } - }, - [ - ActionValues, - method_values(Method, ActionType) - ] - ). - --spec method_values(http_method(), atom()) -> schema_example_map(). -method_values(post, Type) -> +-spec method_values(action | source, http_method(), atom()) -> schema_example_map(). +method_values(Kind, post, Type) -> + KindBin = atom_to_binary(Kind), TypeBin = atom_to_binary(Type), #{ - name => <>, + name => <>, type => TypeBin }; -method_values(get, Type) -> +method_values(Kind, get, Type) -> maps:merge( - method_values(post, Type), + method_values(Kind, post, Type), #{ status => <<"connected">>, node_status => [ @@ -156,7 +251,7 @@ method_values(get, Type) -> ] } ); -method_values(put, _Type) -> +method_values(_Kind, put, _Type) -> #{}. api_fields("get_bridge_v2", Type, Fields) -> @@ -175,60 +270,106 @@ api_fields("post_bridge_v2", Type, Fields) -> ] ); api_fields("put_bridge_v2", _Type, Fields) -> + Fields; +api_fields("get_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + emqx_bridge_schema:status_fields(), + Fields + ] + ); +api_fields("post_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + Fields + ] + ); +api_fields("put_source", _Type, Fields) -> Fields. %%====================================================================================== %% HOCON Schema Callbacks %%====================================================================================== -namespace() -> "actions". +namespace() -> "actions_and_sources". tags() -> - [<<"Actions">>]. + [<<"Actions">>, <<"Sources">>]. -dialyzer({nowarn_function, roots/0}). roots() -> - case fields(actions) of - [] -> - [ - {actions, - ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} - ]; - _ -> - [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] - end. + ActionsRoot = + case fields(actions) of + [] -> + [ + {actions, + ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} + ]; + _ -> + [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] + end, + SourcesRoot = + [{sources, ?HOCON(?R_REF(sources), #{importance => ?IMPORTANCE_LOW})}], + ActionsRoot ++ SourcesRoot. fields(actions) -> - registered_schema_fields(); -fields(resource_opts) -> - resource_opts_fields(_Overrides = []). + registered_schema_fields_actions(); +fields(sources) -> + registered_schema_fields_sources(); +fields(action_resource_opts) -> + action_resource_opts_fields(_Overrides = []); +fields(source_resource_opts) -> + source_resource_opts_fields(_Overrides = []). -registered_schema_fields() -> +registered_schema_fields_actions() -> [ Module:fields(action) - || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules() + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_actions() + ]. + +registered_schema_fields_sources() -> + [ + Module:fields(source) + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_sources() ]. desc(actions) -> ?DESC("desc_bridges_v2"); -desc(resource_opts) -> +desc(sources) -> + ?DESC("desc_sources"); +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(source_resource_opts) -> ?DESC(emqx_resource_schema, "resource_opts"); desc(_) -> undefined. --spec types() -> [action_type()]. -types() -> +-spec action_types() -> [action_type()]. +action_types() -> proplists:get_keys(?MODULE:fields(actions)). --spec types_sc() -> ?ENUM([action_type()]). -types_sc() -> - hoconsc:enum(types()). +-spec action_types_sc() -> ?ENUM([action_type()]). +action_types_sc() -> + hoconsc:enum(action_types()). -resource_opts_fields() -> - resource_opts_fields(_Overrides = []). +-spec source_types() -> [source_type()]. +source_types() -> + proplists:get_keys(?MODULE:fields(sources)). -common_resource_opts_subfields() -> +-spec source_types_sc() -> ?ENUM([source_type()]). +source_types_sc() -> + hoconsc:enum(source_types()). + +action_resource_opts_fields() -> + action_resource_opts_fields(_Overrides = []). + +source_resource_opts_fields() -> + source_resource_opts_fields(_Overrides = []). + +common_action_resource_opts_subfields() -> [ batch_size, batch_time, @@ -244,28 +385,31 @@ common_resource_opts_subfields() -> worker_pool_size ]. -common_resource_opts_subfields_bin() -> - lists:map(fun atom_to_binary/1, common_resource_opts_subfields()). +common_source_resource_opts_subfields() -> + [ + health_check_interval, + resume_interval + ]. -resource_opts_fields(Overrides) -> - ActionROFields = common_resource_opts_subfields(), +common_action_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_action_resource_opts_subfields()). + +common_source_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_source_resource_opts_subfields()). + +action_resource_opts_fields(Overrides) -> + ActionROFields = common_action_resource_opts_subfields(), lists:filter( fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, emqx_resource_schema:create_opts(Overrides) ). -examples(Method) -> - MergeFun = - fun(Example, Examples) -> - maps:merge(Examples, Example) - end, - Fun = - fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), - lists:foldl(MergeFun, Examples, ConnectorExamples) - end, - SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules()], - lists:foldl(Fun, #{}, SchemaModules). +source_resource_opts_fields(Overrides) -> + ActionROFields = common_source_resource_opts_subfields(), + lists:filter( + fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, + emqx_resource_schema:create_opts(Overrides) + ). top_level_common_action_keys() -> [ @@ -286,16 +430,34 @@ make_producer_action_schema(ActionParametersRef) -> make_producer_action_schema(ActionParametersRef, _Opts = #{}). make_producer_action_schema(ActionParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, action_resource_opts)), [ {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})} - | make_consumer_action_schema(ActionParametersRef, Opts) - ]. + | common_schema(ActionParametersRef, Opts) + ] ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. -make_consumer_action_schema(ActionParametersRef) -> - make_consumer_action_schema(ActionParametersRef, _Opts = #{}). +make_consumer_action_schema(ParametersRef) -> + make_consumer_action_schema(ParametersRef, _Opts = #{}). -make_consumer_action_schema(ActionParametersRef, Opts) -> - ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, resource_opts)), +make_consumer_action_schema(ParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, source_resource_opts)), + common_schema(ParametersRef, Opts) ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. + +common_schema(ParametersRef, _Opts) -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {connector, @@ -304,16 +466,15 @@ make_consumer_action_schema(ActionParametersRef, Opts) -> })}, {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()}, - {parameters, ActionParametersRef}, - {resource_opts, - mk(ResourceOptsRef, #{ - default => #{}, - desc => ?DESC(emqx_resource_schema, "resource_opts") - })} + {parameters, ParametersRef} ]. project_to_actions_resource_opts(OldResourceOpts) -> - Subfields = common_resource_opts_subfields_bin(), + Subfields = common_action_resource_opts_subfields_bin(), + maps:with(Subfields, OldResourceOpts). + +project_to_sources_resource_opts(OldResourceOpts) -> + Subfields = common_source_resource_opts_subfields_bin(), maps:with(Subfields, OldResourceOpts). actions_convert_from_connectors(RawConf = #{<<"actions">> := Actions}) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index 30107d0ce..19ae516ec 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -32,7 +32,9 @@ init_per_suite(Config) -> emqx_conf, emqx_connector, emqx_bridge_http, - emqx_bridge + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine ], #{work_dir => ?config(priv_dir, Config)} ), @@ -154,14 +156,18 @@ setup_fake_telemetry_data() -> ok. t_update_ssl_conf(Config) -> - Path = proplists:get_value(config_path, Config), - CertDir = filename:join([emqx:mutable_certs_dir() | Path]), + [_Root, Type, Name] = proplists:get_value(config_path, Config), + CertDir = filename:join([emqx:mutable_certs_dir(), connectors, Type, Name]), EnableSSLConf = #{ <<"bridge_mode">> => false, <<"clean_start">> => true, <<"keepalive">> => <<"60s">>, <<"proto_ver">> => <<"v4">>, <<"server">> => <<"127.0.0.1:1883">>, + <<"egress">> => #{ + <<"local">> => #{<<"topic">> => <<"t">>}, + <<"remote">> => #{<<"topic">> => <<"remote/t">>} + }, <<"ssl">> => #{ <<"cacertfile">> => cert_file("cafile"), @@ -171,10 +177,15 @@ t_update_ssl_conf(Config) -> <<"verify">> => <<"verify_peer">> } }, - {ok, _} = emqx:update_config(Path, EnableSSLConf), + CreateCfg = [ + {bridge_name, Name}, + {bridge_type, Type}, + {bridge_config, #{}} + ], + {ok, _} = emqx_bridge_testlib:create_bridge_api(CreateCfg, EnableSSLConf), ?assertMatch({ok, [_, _, _]}, file:list_dir(CertDir)), NoSSLConf = EnableSSLConf#{<<"ssl">> := #{<<"enable">> => false}}, - {ok, _} = emqx:update_config(Path, NoSSLConf), + {ok, _} = emqx_bridge_testlib:update_bridge_api(CreateCfg, NoSSLConf), {ok, _} = emqx_tls_certfile_gc:force(), ?assertMatch({error, enoent}, file:list_dir(CertDir)), ok. diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 82efc77d2..112e24e63 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -160,8 +160,9 @@ end_per_group(_, Config) -> init_per_testcase(t_broken_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), - meck:expect(emqx_bpapi, supported_version, 1, -1), meck:expect(emqx_bpapi, supported_version, 2, -1), + meck:new(emqx_bridge_api, [passthrough]), + meck:expect(emqx_bridge_api, supported_versions, 1, []), init_per_testcase(common, Config); init_per_testcase(t_old_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), @@ -173,10 +174,10 @@ init_per_testcase(_, Config) -> [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config]. end_per_testcase(t_broken_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(t_old_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(_, Config) -> Sock = ?config(sock, Config), @@ -188,18 +189,7 @@ end_per_testcase(_, Config) -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_bridge:remove(Type, Name) @@ -1026,9 +1016,11 @@ t_with_redact_update(Config) -> BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), + %% bridge is migrated after creation + ConfigRootKey = connectors, ?assertEqual( Password, - get_raw_config([bridges, Type, Name, password], Config) + get_raw_config([ConfigRootKey, Type, Name, password], Config) ), %% probe with new password; should not be considered redacted diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index df4560e6e..7ab8c68a6 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -196,20 +196,10 @@ delete_bridge_http_api_v1(Opts) -> op_bridge_api(Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, + Method = post, + Params = [], + Res = emqx_bridge_v2_testlib:request(Method, Path, Params), ct:pal("bridge op result: ~p", [Res]), Res. diff --git a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl index dadc0a09c..b67791cb3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl @@ -104,7 +104,7 @@ setup_mocks() -> catch meck:new(emqx_bridge_v2_schema, MeckOpts), meck:expect( emqx_bridge_v2_schema, - registered_api_schemas, + registered_actions_api_schemas, 1, fun(Method) -> [{bridge_type_bin(), hoconsc:ref(?MODULE, "api_v2_" ++ Method)}] diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 0c34610ea..fc9c9573f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -24,9 +24,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). --define(ROOT, "actions"). +-define(ACTIONS_ROOT, "actions"). +-define(SOURCES_ROOT, "sources"). --define(CONNECTOR_NAME, <<"my_connector">>). +-define(ACTION_CONNECTOR_NAME, <<"my_connector">>). +-define(SOURCE_CONNECTOR_NAME, <<"my_connector">>). -define(RESOURCE(NAME, TYPE), #{ <<"enable">> => true, @@ -35,10 +37,10 @@ <<"name">> => NAME }). --define(CONNECTOR_TYPE_STR, "kafka_producer"). --define(CONNECTOR_TYPE, <>). +-define(ACTION_CONNECTOR_TYPE_STR, "kafka_producer"). +-define(ACTION_CONNECTOR_TYPE, <>). -define(KAFKA_BOOTSTRAP_HOST, <<"127.0.0.1:9092">>). --define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?CONNECTOR_TYPE)#{ +-define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?ACTION_CONNECTOR_TYPE)#{ <<"authentication">> => <<"none">>, <<"bootstrap_hosts">> => BootstrapHosts, <<"connect_timeout">> => <<"5s">>, @@ -53,14 +55,14 @@ } }). --define(CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). --define(CONNECTOR, ?CONNECTOR(?CONNECTOR_NAME)). +-define(ACTIONS_CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). +-define(ACTIONS_CONNECTOR, ?ACTIONS_CONNECTOR(?ACTION_CONNECTOR_NAME)). -define(MQTT_LOCAL_TOPIC, <<"mqtt/local/topic">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). --define(BRIDGE_TYPE_STR, "kafka_producer"). --define(BRIDGE_TYPE, <>). --define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?BRIDGE_TYPE)#{ +-define(ACTION_TYPE_STR, "kafka_producer"). +-define(ACTION_TYPE, <>). +-define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?ACTION_TYPE)#{ <<"connector">> => Connector, <<"kafka">> => #{ <<"buffer">> => #{ @@ -99,12 +101,15 @@ <<"health_check_interval">> => <<"32s">> } }). --define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?ACTION_CONNECTOR_NAME)). -define(KAFKA_BRIDGE_UPDATE(Name, Connector), maps:without([<<"name">>, <<"type">>], ?KAFKA_BRIDGE(Name, Connector)) ). --define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?ACTION_CONNECTOR_NAME)). + +-define(SOURCE_TYPE_STR, "mqtt"). +-define(SOURCE_TYPE, <>). -define(APPSPECS, [ emqx_conf, @@ -120,34 +125,27 @@ {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ). +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + -if(?EMQX_RELEASE_EDITION == ee). %% For now we got only kafka implementing `bridge_v2` and that is enterprise only. all() -> - [ - {group, single}, - {group, cluster_later_join}, - {group, cluster} - ]. + All0 = emqx_common_test_helpers:all(?MODULE), + All = All0 -- matrix_cases(), + Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()), + Groups ++ All. -else. all() -> []. -endif. +matrix_cases() -> + emqx_common_test_helpers:all(?MODULE). + groups() -> - AllTCs = emqx_common_test_helpers:all(?MODULE), - SingleOnlyTests = [ - t_bridges_probe, - t_broken_bridge_config, - t_fix_broken_bridge_config - ], - ClusterLaterJoinOnlyTCs = [ - t_cluster_later_join_metrics - ], - [ - {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, - {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, - {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} - ]. + emqx_common_test_helpers:matrix_to_groups(?MODULE, matrix_cases()). suite() -> [{timetrap, {seconds, 60}}]. @@ -164,10 +162,16 @@ init_per_group(cluster = Name, Config) -> init_per_group(cluster_later_join = Name, Config) -> Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); -init_per_group(Name, Config) -> - WorkDir = filename:join(?config(priv_dir, Config), Name), +init_per_group(single = Group, Config) -> + WorkDir = filename:join(?config(priv_dir, Config), Group), Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), - init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]). + init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]); +init_per_group(actions, Config) -> + [{bridge_kind, action} | Config]; +init_per_group(sources, Config) -> + [{bridge_kind, source} | Config]; +init_per_group(_Group, Config) -> + Config. init_api(Config) -> Node = ?config(node, Config), @@ -193,8 +197,10 @@ end_per_group(Group, Config) when Group =:= cluster_later_join -> ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)); -end_per_group(_, Config) -> +end_per_group(single, Config) -> emqx_cth_suite:stop(?config(group_apps, Config)), + ok; +end_per_group(_Group, _Config) -> ok. init_per_testcase(t_action_types, Config) -> @@ -212,7 +218,17 @@ init_per_testcase(_TestCase, Config) -> Nodes -> [erpc:call(Node, ?MODULE, init_mocks, []) || Node <- Nodes] end, - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR, Config), + case ?config(bridge_kind, Config) of + action -> + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config); + source -> + {ok, 201, _} = request( + post, + uri(["connectors"]), + source_connector_create_config(#{}), + Config + ) + end, Config. end_per_testcase(_TestCase, Config) -> @@ -227,6 +243,10 @@ end_per_testcase(_TestCase, Config) -> ok = emqx_common_test_helpers:call_janitor(), ok. +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + -define(CONNECTOR_IMPL, emqx_bridge_v2_dummy_connector). init_mocks() -> case emqx_release:edition() of @@ -235,6 +255,8 @@ init_mocks() -> meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), ok; ce -> + meck:new(emqx_connector_resource, [passthrough, no_link]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), ok end, meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), @@ -243,7 +265,7 @@ init_mocks() -> ?CONNECTOR_IMPL, on_start, fun - (<<"connector:", ?CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> + (<<"connector:", ?ACTION_CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> {ok, bad_connector_state}; (_I, _C) -> {ok, connector_state} @@ -267,454 +289,7 @@ init_mocks() -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ). - -%%------------------------------------------------------------------------------ -%% Testcases -%%------------------------------------------------------------------------------ - -%% We have to pretend testing a kafka bridge since at this point that's the -%% only one that's implemented. - -t_bridges_lifecycle(Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - {ok, 404, _} = request(get, uri([?ROOT, "foo"]), Config), - {ok, 404, _} = request(get, uri([?ROOT, "kafka_producer:foo"]), Config), - - %% need a var for patterns below - BridgeName = ?BRIDGE_NAME, - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"connector">> := ?CONNECTOR_NAME, - <<"parameters">> := #{}, - <<"local_topic">> := _, - <<"resource_opts">> := _ - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME), - Config - ) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% get the bridge by id - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ?assertMatch( - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := _ - }}, - request_json(post, uri([?ROOT, BridgeID, "brababbel"]), Config) - ), - - %% update bridge config - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR(<<"foobla">>), Config), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"connector">> := <<"foobla">>, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), - Config - ) - ), - - %% update bridge with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message1 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message1) - ), - - %% update bridge with connector of wrong type - {ok, 201, _} = - request( - post, - uri(["connectors"]), - (?CONNECTOR(<<"foobla2">>))#{ - <<"type">> => <<"azure_event_hub_producer">>, - <<"authentication">> => #{ - <<"username">> => <<"emqxuser">>, - <<"password">> => <<"topSecret">>, - <<"mechanism">> => <<"plain">> - }, - <<"ssl">> => #{ - <<"enable">> => true, - <<"server_name_indication">> => <<"auto">>, - <<"verify">> => <<"verify_none">>, - <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] - } - }, - Config - ), - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message2 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message2) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% try create with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message3 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message3) - ), - - %% try create bridge with connector of wrong type - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message4 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message4) - ), - - %% make sure nothing has been created above - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% update a deleted bridge returns an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), - Config - ) - ), - - %% deleting a non-existing bridge should result in an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json(delete, uri([?ROOT, BridgeID]), Config) - ), - - %% try delete unknown bridge id - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := <<"Invalid bridge ID", _/binary>> - }}, - request_json(delete, uri([?ROOT, "foo"]), Config) - ), - - %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"éš‹è¾¾"/utf8>>), Config), - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), - ok. - -t_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertEqual( - {ok, 204, <<>>}, - request(delete, uri([?ROOT, BridgeID]), Config) - ), - - ?assertEqual( - {ok, 200, []}, - request_json(get, uri([?ROOT]), Config) - ), - - ok. - -t_fix_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?CONNECTOR_NAME), - Config - ), - - ?assertMatch( - {ok, 200, #{ - <<"connector">> := ?CONNECTOR_NAME, - <<"status">> := <<"connected">> - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ok. - -t_start_bridge_unknown_node(Config) -> - {ok, 404, _} = - request( - post, - uri(["nodes", "thisbetterbenotanatomyet", ?ROOT, "kafka_producer:foo", start]), - Config - ), - {ok, 404, _} = - request( - post, - uri(["nodes", "undefined", ?ROOT, "kafka_producer:foo", start]), - Config - ). - -t_start_bridge_node(Config) -> - do_start_bridge(node, Config). - -t_start_bridge_cluster(Config) -> - do_start_bridge(cluster, Config). - -do_start_bridge(TestType, Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - Name = atom_to_binary(TestType), - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _] - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(Name), - Config - ) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), - - %% start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - %% start a started bridge - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), - - %% Make start bridge fail - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, - Config - ), - - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, stop), - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, start), - - {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% Make start bridge succeed - - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, - Config - ), - - %% try to start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% Fail parse-id check - {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), - %% Looks ok but doesn't exist - {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), - ok. + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(). expect_on_all_nodes(Mod, Function, Fun, Config) -> case ?config(cluster_nodes, Config) of @@ -751,6 +326,705 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> ok = emqx_connector_resource:OperationName(ConnectorType, ConnectorName) end. +listen_on_random_port() -> + SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], + case gen_tcp:listen(0, SockOpts) of + {ok, Sock} -> + {ok, Port} = inet:port(Sock), + {Port, Sock}; + {error, Reason} when Reason /= eaddrinuse -> + {error, Reason} + end. + +request(Method, URL, Config) -> + request(Method, URL, [], Config). + +request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> + URL = operation_path(Type, Op, BridgeID, Config), + request(Method, URL, Body, Config); +request(Method, URL, Body, Config) -> + AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). + +request(Method, URL, Body, Decoder, Config) -> + case request(Method, URL, Body, Config) of + {ok, Code, Response} -> + case Decoder(Response) of + {error, _} = Error -> Error; + Decoded -> {ok, Code, Decoded} + end; + Otherwise -> + Otherwise + end. + +request_json(Method, URLLike, Config) -> + request(Method, URLLike, [], fun json/1, Config). + +request_json(Method, URLLike, Body, Config) -> + request(Method, URLLike, Body, fun json/1, Config). + +operation_path(node, Oper, BridgeID, Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri(["nodes", ?config(node, Config), APIRootKey, BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri([APIRootKey, BridgeID, Oper]). + +enable_path(Enable, BridgeID) -> + uri([?ACTIONS_ROOT, BridgeID, "enable", Enable]). + +publish_message(Topic, Body, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). + +update_config(Path, Value, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, update_config, [Path, Value]). + +get_raw_config(Path, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, get_raw_config, [Path]). + +add_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). + +delete_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). + +str(S) when is_list(S) -> S; +str(S) when is_binary(S) -> binary_to_list(S). + +json(B) when is_binary(B) -> + case emqx_utils_json:safe_decode(B, [return_maps]) of + {ok, Term} -> + Term; + {error, Reason} = Error -> + ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), + Error + end. + +group_path(Config) -> + case emqx_common_test_helpers:group_path(Config) of + [] -> + undefined; + Path -> + Path + end. + +source_connector_config_base() -> + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_connector_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_connector_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE, + <<"name">> => ?SOURCE_CONNECTOR_NAME + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_config_base() -> + #{ + <<"enable">> => true, + <<"connector">> => ?SOURCE_CONNECTOR_NAME, + <<"parameters">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"resume_interval">> => <<"15s">> + } + }. + +source_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_update_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + maps:merge( + source_config_base(), + Overrides + ). + +get_common_values(Kind, FnName) -> + case Kind of + actions -> + #{ + api_root_key => ?ACTIONS_ROOT, + type => ?ACTION_TYPE, + default_connector_name => ?ACTION_CONNECTOR_NAME, + create_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE(Name, ConnectorName) + end, + update_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE_UPDATE(Name, ConnectorName) + end, + create_connector_config_fn => + fun(Overrides) -> + ConnectorName = maps:get(name, Overrides, ?ACTION_CONNECTOR_NAME), + ?ACTIONS_CONNECTOR(ConnectorName) + end + }; + sources -> + #{ + api_root_key => ?SOURCES_ROOT, + type => ?SOURCE_TYPE, + default_connector_name => ?SOURCE_CONNECTOR_NAME, + create_config_fn => fun(Overrides0) -> + Overrides = + case Overrides0 of + #{name := _} -> Overrides0; + _ -> Overrides0#{name => FnName} + end, + source_create_config(Overrides) + end, + update_config_fn => fun source_update_config/1, + create_connector_config_fn => fun source_connector_create_config/1 + } + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +%% We have to pretend testing a kafka bridge since at this point that's the +%% only one that's implemented. + +t_bridges_lifecycle(matrix) -> + [ + [single, actions], + [single, sources], + [cluster, actions], + [cluster, sources] + ]; +t_bridges_lifecycle(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + FnName = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn, + update_config_fn := UpdateConfigFn, + create_connector_config_fn := CreateConnectorConfigFn + } = get_common_values(Kind, FnName), + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + + {ok, 404, _} = request(get, uri([APIRootKey, "foo"]), Config), + {ok, 404, _} = request(get, uri([APIRootKey, "kafka_producer:foo"]), Config), + + %% need a var for patterns below + BridgeName = FnName, + CreateRes = request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{}), + Config + ), + ?assertMatch( + {ok, 201, #{ + <<"type">> := Type, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"connector">> := DefaultConnectorName, + <<"parameters">> := #{}, + <<"resource_opts">> := _ + }}, + CreateRes, + #{name => BridgeName, type => Type, connector => DefaultConnectorName} + ), + case Kind of + actions -> + ?assertMatch({ok, 201, #{<<"local_topic">> := _}}, CreateRes); + sources -> + ok + end, + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := Type, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([APIRootKey]), Config) + ), + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := Type, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([APIRootKey]), Config) + ), + + %% get the bridge by id + BridgeID = emqx_bridge_resource:bridge_id(Type, ?BRIDGE_NAME), + ?assertMatch( + {ok, 200, #{ + <<"type">> := Type, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json(get, uri([APIRootKey, BridgeID]), Config) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := _ + }}, + request_json(post, uri([APIRootKey, BridgeID, "brababbel"]), Config) + ), + + %% update bridge config + {ok, 201, _} = request( + post, + uri(["connectors"]), + CreateConnectorConfigFn(#{name => <<"foobla">>}), + Config + ), + ?assertMatch( + {ok, 200, #{ + <<"type">> := Type, + <<"name">> := BridgeName, + <<"connector">> := <<"foobla">>, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json( + put, + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla">>}), + Config + ) + ), + + %% update bridge with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message1 + }} = + request_json( + put, + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"does_not_exist">>}), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message1) + ), + + %% update bridge with connector of wrong type + {ok, 201, _} = + request( + post, + uri(["connectors"]), + (?ACTIONS_CONNECTOR(<<"foobla2">>))#{ + <<"type">> => <<"azure_event_hub_producer">>, + <<"authentication">> => #{ + <<"username">> => <<"emqxuser">>, + <<"password">> => <<"topSecret">>, + <<"mechanism">> => <<"plain">> + }, + <<"ssl">> => #{ + <<"enable">> => true, + <<"server_name_indication">> => <<"auto">>, + <<"verify">> => <<"verify_none">>, + <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] + } + }, + Config + ), + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message2 + }} = + request_json( + put, + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla2">>}), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message2) + ), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + + %% try create with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message3 + }} = + request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"does_not_exist">>}), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message3) + ), + + %% try create bridge with connector of wrong type + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message4 + }} = + request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"foobla2">>}), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message4) + ), + + %% make sure nothing has been created above + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + + %% update a deleted bridge returns an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json( + put, + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{}), + Config + ) + ), + + %% deleting a non-existing bridge should result in an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json(delete, uri([APIRootKey, BridgeID]), Config) + ), + + %% try delete unknown bridge id + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Invalid bridge ID", _/binary>> + }}, + request_json(delete, uri([APIRootKey, "foo"]), Config) + ), + + %% Try create bridge with bad characters as name + {ok, 400, _} = request( + post, uri([APIRootKey]), CreateConfigFn(#{name => <<"éš‹è¾¾"/utf8>>}), Config + ), + {ok, 400, _} = request(post, uri([APIRootKey]), CreateConfigFn(#{name => <<"a.b">>}), Config), + ok. + +t_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + ?assertEqual( + {ok, 204, <<>>}, + request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ?assertEqual( + {ok, 200, []}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + ok. + +t_fix_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_fix_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?ACTION_CONNECTOR_NAME), + Config + ), + + ?assertMatch( + {ok, 200, #{ + <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"status">> := <<"connected">> + }}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ok. + +t_start_bridge_unknown_node(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_unknown_node(Config) -> + {ok, 404, _} = + request( + post, + uri(["nodes", "thisbetterbenotanatomyet", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ), + {ok, 404, _} = + request( + post, + uri(["nodes", "undefined", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ). + +t_start_bridge_node(matrix) -> + [ + [single, actions], + [single, sources], + [cluster, actions], + [cluster, sources] + ]; +t_start_bridge_node(Config) -> + do_start_bridge(node, Config). + +t_start_bridge_cluster(matrix) -> + [ + [single, actions], + [single, sources], + [cluster, actions], + [cluster, sources] + ]; +t_start_bridge_cluster(Config) -> + do_start_bridge(cluster, Config). + +do_start_bridge(TestType, Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + Name = atom_to_binary(TestType), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + + ?assertMatch( + {ok, 201, #{ + <<"type">> := Type, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _] + }}, + request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{name => Name}), + Config + ) + ), + + BridgeID = emqx_bridge_resource:bridge_id(Type, Name), + + %% start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([APIRootKey, BridgeID]), Config) + ), + %% start a started bridge + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([APIRootKey, BridgeID]), Config) + ), + + {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), + + %% Make start bridge fail + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, + Config + ), + + connector_operation(Config, Type, DefaultConnectorName, stop), + connector_operation(Config, Type, DefaultConnectorName, start), + + {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% Make start bridge succeed + + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, + Config + ), + + %% try to start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + + %% Fail parse-id check + {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), + %% Looks ok but doesn't exist + {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), + ok. + %% t_start_stop_inconsistent_bridge_node(Config) -> %% start_stop_inconsistent_bridge(node, Config). @@ -861,6 +1135,10 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> %% {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), %% {ok, 200, []} = request_json(get, uri([?ROOT]), Config). +t_bridges_probe(matrix) -> + [ + [single, actions] + ]; t_bridges_probe(Config) -> {ok, 204, <<>>} = request( post, @@ -905,15 +1183,20 @@ t_bridges_probe(Config) -> ), ok. +t_cascade_delete_actions(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_cascade_delete_actions(Config) -> %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% then we add a a bridge, using POST %% POST /actions/ will create a bridge - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -931,10 +1214,10 @@ t_cascade_delete_actions(Config) -> %% delete the bridge will also delete the actions from the rules {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertMatch( {ok, 200, #{<<"actions">> := []}}, request_json(get, uri(["rules", RuleId]), Config) @@ -943,7 +1226,7 @@ t_cascade_delete_actions(Config) -> {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -960,19 +1243,24 @@ t_cascade_delete_actions(Config) -> ), {ok, 400, Body} = request( delete, - uri([?ROOT, BridgeID]), + uri([?ACTIONS_ROOT, BridgeID]), Config ), ?assertMatch(#{<<"rules">> := [_ | _]}, emqx_utils_json:decode(Body, [return_maps])), - {ok, 200, [_]} = request_json(get, uri([?ROOT]), Config), + {ok, 200, [_]} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% Cleanup {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config). + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config). +t_action_types(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_action_types(Config) -> Res = request_json(get, uri(["action_types"]), Config), ?assertMatch({ok, 200, _}, Res), @@ -981,12 +1269,24 @@ t_action_types(Config) -> ?assert(lists:all(fun is_binary/1, Types), #{types => Types}), ok. +t_bad_name(matrix) -> + [ + [single, actions], + [single, sources], + [cluster, actions], + [cluster, sources] + ]; t_bad_name(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), Name = <<"_bad_name">>, + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), Res = request_json( post, - uri([?ROOT]), - ?KAFKA_BRIDGE(Name), + uri([APIRootKey]), + CreateConfigFn(#{}), Config ), ?assertMatch({ok, 400, #{<<"message">> := _}}, Res), @@ -1001,31 +1301,36 @@ t_bad_name(Config) -> ), ok. +t_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_metrics(Config) -> - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 0}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), - {ok, 200, Bridge} = request_json(get, uri([?ROOT, ActionID]), Config), + {ok, 200, Bridge} = request_json(get, uri([?ACTIONS_ROOT, ActionID]), Config), ?assertNot(maps:is_key(<<"metrics">>, Bridge)), ?assertNot(maps:is_key(<<"node_metrics">>, Bridge)), @@ -1041,12 +1346,12 @@ t_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 1}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), %% check for absence of metrics when listing all bridges - {ok, 200, Bridges} = request_json(get, uri([?ROOT]), Config), + {ok, 200, Bridges} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertNotMatch( [ #{ @@ -1058,21 +1363,26 @@ t_metrics(Config) -> ), ok. +t_reset_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_reset_metrics(Config) -> %% assert there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), Body = <<"my msg">>, _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), @@ -1084,11 +1394,11 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), - {ok, 204, <<>>} = request(put, uri([?ROOT, ActionID, "metrics", "reset"]), Config), + {ok, 204, <<>>} = request(put, uri([?ACTIONS_ROOT, ActionID, "metrics", "reset"]), Config), ?retry( _Sleep0 = 200, @@ -1098,28 +1408,34 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), ok. +t_cluster_later_join_metrics(matrix) -> + [ + [cluster_later_join, actions] + ]; t_cluster_later_join_metrics(Config) -> [PrimaryNode, OtherNode | _] = ?config(cluster_nodes, Config), Name = ?BRIDGE_NAME, ActionParams = ?KAFKA_BRIDGE(Name), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), ?check_trace( begin %% Create a bridge on only one of the nodes. - ?assertMatch({ok, 201, _}, request_json(post, uri([?ROOT]), ActionParams, Config)), + ?assertMatch( + {ok, 201, _}, request_json(post, uri([?ACTIONS_ROOT]), ActionParams, Config) + ), %% Pre-condition. ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), %% Now join the other node join with the api node. ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]), @@ -1130,7 +1446,7 @@ t_cluster_later_join_metrics(Config) -> <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), ok end, @@ -1138,94 +1454,28 @@ t_cluster_later_join_metrics(Config) -> ), ok. +t_raw_config_response_defaults(matrix) -> + [ + [single, actions], + [single, sources], + [cluster, actions], + [cluster, sources] + ]; t_raw_config_response_defaults(Config) -> - Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), + [_SingleOrCluster, Kind | _] = group_path(Config), + Name = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + Params = maps:remove(<<"enable">>, CreateConfigFn(#{})), ?assertMatch( {ok, 201, #{<<"enable">> := true}}, request_json( post, - uri([?ROOT]), + uri([APIRootKey]), Params, Config ) ), ok. - -%%% helpers -listen_on_random_port() -> - SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], - case gen_tcp:listen(0, SockOpts) of - {ok, Sock} -> - {ok, Port} = inet:port(Sock), - {Port, Sock}; - {error, Reason} when Reason /= eaddrinuse -> - {error, Reason} - end. - -request(Method, URL, Config) -> - request(Method, URL, [], Config). - -request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> - URL = operation_path(Type, Op, BridgeID, Config), - request(Method, URL, Body, Config); -request(Method, URL, Body, Config) -> - AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), - Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, - emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). - -request(Method, URL, Body, Decoder, Config) -> - case request(Method, URL, Body, Config) of - {ok, Code, Response} -> - case Decoder(Response) of - {error, _} = Error -> Error; - Decoded -> {ok, Code, Decoded} - end; - Otherwise -> - Otherwise - end. - -request_json(Method, URLLike, Config) -> - request(Method, URLLike, [], fun json/1, Config). - -request_json(Method, URLLike, Body, Config) -> - request(Method, URLLike, Body, fun json/1, Config). - -operation_path(node, Oper, BridgeID, Config) -> - uri(["nodes", ?config(node, Config), ?ROOT, BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID, _Config) -> - uri([?ROOT, BridgeID, Oper]). - -enable_path(Enable, BridgeID) -> - uri([?ROOT, BridgeID, "enable", Enable]). - -publish_message(Topic, Body, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). - -update_config(Path, Value, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, update_config, [Path, Value]). - -get_raw_config(Path, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, get_raw_config, [Path]). - -add_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). - -delete_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). - -str(S) when is_list(S) -> S; -str(S) when is_binary(S) -> binary_to_list(S). - -json(B) when is_binary(B) -> - case emqx_utils_json:safe_decode(B, [return_maps]) of - {ok, Term} -> - Term; - {error, Reason} = Error -> - ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), - Error - end. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index eb8a9a5f8..7fef33115 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -12,6 +12,9 @@ -import(emqx_common_test_helpers, [on_exit/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% ct setup helpers init_per_suite(Config, Apps) -> @@ -96,9 +99,15 @@ delete_all_bridges_and_connectors() -> delete_all_bridges() -> lists:foreach( fun(#{name := Name, type := Type}) -> - emqx_bridge_v2:remove(Type, Name) + emqx_bridge_v2:remove(actions, Type, Name) end, - emqx_bridge_v2:list() + emqx_bridge_v2:list(actions) + ), + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge_v2:remove(sources, Type, Name) + end, + emqx_bridge_v2:list(sources) ). delete_all_connectors() -> @@ -146,6 +155,49 @@ create_bridge(Config, Overrides) -> ct:pal("creating bridge with config: ~p", [BridgeConfig]), emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig). +get_ct_config_with_fallback(Config, [Key]) -> + ?config(Key, Config); +get_ct_config_with_fallback(Config, [Key | Rest]) -> + case ?config(Key, Config) of + undefined -> + get_ct_config_with_fallback(Config, Rest); + X -> + X + end. + +get_config_by_kind(Config, Overrides) -> + Kind = ?config(bridge_kind, Config), + get_config_by_kind(Kind, Config, Overrides). + +get_config_by_kind(Kind, Config, Overrides) -> + case Kind of + action -> + %% TODO: refactor tests to use action_type... + ActionType = get_ct_config_with_fallback(Config, [action_type, bridge_type]), + ActionName = get_ct_config_with_fallback(Config, [action_name, bridge_name]), + ActionConfig0 = get_ct_config_with_fallback(Config, [action_config, bridge_config]), + ActionConfig = emqx_utils_maps:deep_merge(ActionConfig0, Overrides), + #{type => ActionType, name => ActionName, config => ActionConfig}; + source -> + SourceType = ?config(source_type, Config), + SourceName = ?config(source_name, Config), + SourceConfig0 = ?config(source_config, Config), + SourceConfig = emqx_utils_maps:deep_merge(SourceConfig0, Overrides), + #{type => SourceType, name => SourceName, config => SourceConfig} + end. + +api_path_root(Kind) -> + case Kind of + action -> "actions"; + source -> "sources" + end. + +conf_root_key(Kind) -> + case Kind of + action -> ?ROOT_KEY_ACTIONS; + source -> ?ROOT_KEY_SOURCES + end. + maybe_json_decode(X) -> case emqx_utils_json:safe_decode(X, [return_maps]) of {ok, Decoded} -> Decoded; @@ -212,26 +264,26 @@ create_bridge_api(Config) -> create_bridge_api(Config, _Overrides = #{}). create_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - {ok, {{_, 201, _}, _, _}} = create_connector_api(Config), + create_kind_api(Config, Overrides). - Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("creating bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge create result: ~p", [Res]), +create_kind_api(Config) -> + create_kind_api(Config, _Overrides = #{}). + +create_kind_api(Config, Overrides) -> + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, Overrides), + Params = BridgeConfig#{<<"type">> => Type, <<"name">> => Name}, + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot]), + ct:pal("creating bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge create (~s, http) result:\n ~p", [Kind, Res]), Res. create_connector_api(Config) -> @@ -282,41 +334,33 @@ update_bridge_api(Config) -> update_bridge_api(Config, _Overrides = #{}). update_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - Name = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - BridgeId = emqx_bridge_resource:bridge_id(BridgeType, Name), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("updating bridge (via http): ~p", [BridgeConfig]), - Res = - case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, BridgeConfig, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; - Error -> Error - end, - ct:pal("bridge update result: ~p", [Res]), + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := Params + } = get_config_by_kind(Kind, Config, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId]), + ct:pal("updating bridge (~s, http):\n ~p", [Kind, Params]), + Method = put, + Res = request(Method, Path, Params), + ct:pal("update bridge (~s, http) result:\n ~p", [Kind, Res]), Res. op_bridge_api(Op, BridgeType, BridgeName) -> + op_bridge_api(_Kind = action, Op, BridgeType, BridgeName). + +op_bridge_api(Kind, Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge op result: ~p", [Res]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId, Op]), + ct:pal("calling bridge ~p (~s, http):\n ~p", [BridgeId, Kind, Op]), + Method = post, + Params = [], + Res = request(Method, Path, Params), + ct:pal("bridge op result:\n ~p", [Res]), Res. probe_bridge_api(Config) -> @@ -330,17 +374,16 @@ probe_bridge_api(Config, Overrides) -> probe_bridge_api(BridgeType, BridgeName, BridgeConfig). probe_bridge_api(BridgeType, BridgeName, BridgeConfig) -> + probe_bridge_api(action, BridgeType, BridgeName, BridgeConfig). + +probe_bridge_api(Kind, BridgeType, BridgeName, BridgeConfig) -> Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions_probe"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("probing bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; - Error -> Error - end, - ct:pal("bridge probe result: ~p", [Res]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot ++ "_probe"]), + ct:pal("probing bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge probe (~s, http) result:\n ~p", [Kind, Res]), Res. list_bridges_http_api_v1() -> @@ -357,6 +400,13 @@ list_actions_http_api() -> ct:pal("list actions (http v2) result:\n ~p", [Res]), Res. +list_sources_http_api() -> + Path = emqx_mgmt_api_test_util:api_path(["sources"]), + ct:pal("list sources (http v2)"), + Res = request(get, Path, _Params = []), + ct:pal("list sources (http v2) result:\n ~p", [Res]), + Res. + list_connectors_http_api() -> Path = emqx_mgmt_api_test_util:api_path(["connectors"]), ct:pal("list connectors"), @@ -392,6 +442,23 @@ try_decode_error(Body0) -> Body0 end. +create_rule_api(Opts) -> + #{ + sql := SQL, + actions := RuleActions + } = Opts, + Params = #{ + enable => true, + sql => SQL, + actions => RuleActions + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + ct:pal("create rule:\n ~p", [Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("create rule results:\n ~p", [Res]), + Res. + create_rule_and_action_http(BridgeType, RuleTopic, Config) -> create_rule_and_action_http(BridgeType, RuleTopic, Config, _Opts = #{}). @@ -510,13 +577,6 @@ t_create_via_http(Config) -> begin ?assertMatch({ok, _}, create_bridge_api(Config)), - %% lightweight matrix testing some configs - ?assertMatch( - {ok, _}, - update_bridge_api( - Config - ) - ), ?assertMatch( {ok, _}, update_bridge_api( @@ -530,23 +590,26 @@ t_create_via_http(Config) -> ok. t_start_stop(Config, StopTracePoint) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig = ?config(bridge_config, Config), + Kind = proplists:get_value(bridge_kind, Config, action), ConnectorName = ?config(connector_name, Config), ConnectorType = ?config(connector_type, Config), - ConnectorConfig = ?config(connector_config, Config), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, _Overrides = #{}), ?assertMatch( - {ok, _}, - emqx_connector:create(ConnectorType, ConnectorName, ConnectorConfig) + {ok, {{_, 201, _}, _, _}}, + create_connector_api(Config) ), ?check_trace( begin ProbeRes0 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), @@ -554,8 +617,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsBefore = erlang:system_info(atom_count), %% Probe again; shouldn't have created more atoms. ProbeRes1 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), @@ -563,9 +627,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), - ?assertMatch({ok, _}, emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig)), + ?assertMatch({ok, _}, create_kind_api(Config)), - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + ResourceId = emqx_bridge_resource:resource_id(conf_root_key(Kind), Type, Name), %% Since the connection process is async, we give it some time to %% stabilize and avoid flakiness. @@ -578,7 +642,7 @@ t_start_stop(Config, StopTracePoint) -> %% `start` bridge to trigger `already_started` ?assertMatch( {ok, {{_, 204, _}, _Headers, []}}, - emqx_bridge_v2_testlib:op_bridge_api("start", BridgeType, BridgeName) + op_bridge_api(Kind, "start", Type, Name) ), ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), @@ -628,10 +692,10 @@ t_start_stop(Config, StopTracePoint) -> ) ), - ok + #{resource_id => ResourceId} end, - fun(Trace) -> - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + fun(Res, Trace) -> + #{resource_id := ResourceId} = Res, %% one for each probe, one for real ?assertMatch( [_, _, #{instance_id := ResourceId}], diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index e90100995..a6c66c609 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -48,7 +48,9 @@ resource_opts_union_connector_actions_test() -> %% consciouly between connector and actions, in particular when/if we introduce new %% fields there. AllROFields = non_deprecated_fields(emqx_resource_schema:create_opts([])), - ActionROFields = non_deprecated_fields(emqx_bridge_v2_schema:resource_opts_fields()), + ActionROFields = non_deprecated_fields( + emqx_bridge_v2_schema:action_resource_opts_fields() + ), ConnectorROFields = non_deprecated_fields(emqx_connector_schema:resource_opts_fields()), UnionROFields = lists:usort(ConnectorROFields ++ ActionROFields), ?assertEqual( @@ -108,7 +110,7 @@ connector_resource_opts_test() -> ok. actions_api_spec_post_fields_test() -> - ?UNION(Union) = emqx_bridge_v2_schema:post_request(), + ?UNION(Union) = emqx_bridge_v2_schema:actions_post_request(), Schemas = lists:map( fun(?R_REF(SchemaMod, StructName)) -> diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl index a7efebf89..2ea7fed22 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl @@ -72,7 +72,7 @@ parse(Hocon) -> Conf. check(SchemaMod, Conf) when is_map(Conf) -> - hocon_tconf:check_plain(SchemaMod, Conf). + hocon_tconf:check_plain(SchemaMod, Conf, #{required => false}). check_action(Conf) when is_map(Conf) -> check(emqx_bridge_v2_schema, Conf). diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index 032439574..97f3986e4 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -52,7 +52,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_create) -> [ diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl index e030bbc74..92e4e09a6 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl @@ -159,7 +159,7 @@ generate_config(Config0) -> } = gcp_pubsub_config(Config0), %% FIXME %% `emqx_bridge_resource:resource_id' requires an existing connector in the config..... - ConnectorName = <<"connector_", ActionName/binary>>, + ConnectorName = ActionName, ConnectorResourceId = <<"connector:", ?CONNECTOR_TYPE_BIN/binary, ":", ConnectorName/binary>>, ActionResourceId = emqx_bridge_v2:id(?ACTION_TYPE_BIN, ActionName, ConnectorName), BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_V1_TYPE_BIN, ActionName), @@ -1228,7 +1228,11 @@ do_econnrefused_or_timeout_test(Config, Error) -> %% _Msg = "The connection was lost." ok; Trace0 -> - error({unexpected_trace, Trace0}) + error( + {unexpected_trace, Trace0, #{ + expected_connector_id => ConnectorResourceId + }} + ) end; timeout -> ?assertMatch( diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index 009eb75e6..e2ecdd3dc 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -106,7 +106,7 @@ fields(action_resource_opts) -> UnsupportedOpts = [batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields("parameters_opts") -> [ diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 932191ec5..1874fc72e 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -244,6 +244,12 @@ parse_http_request_assertive(ReqStr0) -> %% Helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +get_metrics(Name) -> + %% Note: `emqx_bridge:get_metrics/2' is currently *only* called in prod by + %% `emqx_bridge_api:lookup_from_local_node' with an action (not v1 bridge) type. + Type = <<"http">>, + emqx_bridge:get_metrics(Type, Name). + bridge_async_config(#{port := Port} = Config) -> Type = maps:get(type, Config, ?BRIDGE_TYPE), Name = maps:get(name, Config, ?BRIDGE_NAME), @@ -570,7 +576,7 @@ t_path_not_found(Config) -> success := 0 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -611,7 +617,7 @@ t_too_many_requests(Config) -> success := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -654,7 +660,7 @@ t_rule_action_expired(Config) -> dropped := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ?retry( diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 4be7feb19..c33ea757b 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -70,7 +70,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_parameters) -> [ diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 061543b2b..b1032ff6b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -553,7 +553,7 @@ fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(resource_opts) -> SupportedFields = [health_check_interval], - CreationOpts = emqx_bridge_v2_schema:resource_opts_fields(), + CreationOpts = emqx_bridge_v2_schema:action_resource_opts_fields(), lists:filter(fun({Field, _}) -> lists:member(Field, SupportedFields) end, CreationOpts); fields(action_field) -> {kafka_producer, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 7caab1d87..459e259d2 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -135,7 +135,7 @@ create_producers_for_bridge_v2( KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), - {_BridgeType, BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), + #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), IsDryRun = case TestIdStart of diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index e8eb93624..ba7be1ec4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -97,7 +97,7 @@ fields(action_parameters) -> fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{ importance => ?IMPORTANCE_HIDDEN, converter => fun(_, _) -> 1 end, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e6fe78ab8..0c00a0d59 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index cc2296d3c..9aae73bd2 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -20,8 +20,13 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). +-behaviour(ecpool_worker). + +%% ecpool +-export([connect/1]). -export([on_message_received/3]). +-export([handle_disconnect/1]). %% callbacks of behaviour emqx_resource -export([ @@ -30,11 +35,25 @@ on_stop/2, on_query/3, on_query_async/4, - on_get_status/2 + on_get_status/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channel_status/3, + on_get_channels/1 ]). -export([on_async_result/2]). +-type name() :: term(). + +-type option() :: + {name, name()} + | {ingress, map()} + %% see `emqtt:option()` + | {client_opts, map()}. + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -define(HEALTH_CHECK_TIMEOUT, 1000). -define(INGRESS, "I"). -define(EGRESS, "E"). @@ -42,142 +61,211 @@ %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, HookPoint, ResId) -> + +on_message_received(Msg, HookPoints, ResId) -> emqx_resource_metrics:received_inc(ResId), - emqx_hooks:run(HookPoint, [Msg]). + lists:foreach( + fun(HookPoint) -> + emqx_hooks:run(HookPoint, [Msg]) + end, + HookPoints + ), + ok. %% =================================================================== callback_mode() -> async_if_possible. -on_start(ResourceId, Conf) -> +on_start(ResourceId, #{server := Server} = Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => ResourceId, config => emqx_utils:redact(Conf) }), - case start_ingress(ResourceId, Conf) of + TopicToHandlerIndex = emqx_topic_index:new(), + StartConf = Conf#{topic_to_handler_index => TopicToHandlerIndex}, + case start_mqtt_clients(ResourceId, StartConf) of {ok, Result1} -> - case start_egress(ResourceId, Conf) of - {ok, Result2} -> - {ok, maps:merge(Result1, Result2)}; - {error, Reason} -> - _ = stop_ingress(Result1), - {error, Reason} - end; + {ok, Result1#{ + installed_channels => #{}, + clean_start => maps:get(clean_start, Conf), + topic_to_handler_index => TopicToHandlerIndex, + server => Server + }}; {error, Reason} -> {error, Reason} end. -start_ingress(ResourceId, Conf) -> - ClientOpts = mk_client_opts(ResourceId, ?INGRESS, Conf), - case mk_ingress_config(ResourceId, Conf) of - Ingress = #{} -> - start_ingress(ResourceId, Ingress, ClientOpts); - undefined -> - {ok, #{}} - end. - -start_ingress(ResourceId, Ingress, ClientOpts) -> - PoolName = <>, - PoolSize = choose_ingress_pool_size(ResourceId, Ingress), - Options = [ - {name, PoolName}, - {pool_size, PoolSize}, - {ingress, Ingress}, - {client_opts, ClientOpts} - ], - ok = emqx_resource:allocate_resource(ResourceId, ingress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_ingress, Options) of - ok -> - {ok, #{ingress_pool_name => PoolName}}; - {error, {start_pool_failed, _, Reason}} -> - {error, Reason} - end. - -choose_ingress_pool_size(<>, _) -> - 1; -choose_ingress_pool_size( - ResourceId, - #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} +on_add_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + clean_start := CleanStart + } = OldState, + ChannelId, + #{config_root := actions} = ChannelConfig ) -> - case emqx_topic:parse(RemoteTopic) of - {#share{} = _Filter, _SubOpts} -> - % NOTE: this is shared subscription, many workers may subscribe - PoolSize; - {_Filter, #{}} when PoolSize > 1 -> - % NOTE: this is regular subscription, only one worker should subscribe + %% Publisher channel + %% make a warning if clean_start is set to false + case CleanStart of + false -> + ?tp( + mqtt_clean_start_egress_action_warning, + #{ + channel_id => ChannelId, + resource_id => _InstId + } + ), ?SLOG(warning, #{ - msg => "mqtt_bridge_ingress_pool_size_ignored", - connector => ResourceId, - reason => - "Remote topic filter is not a shared subscription, " - "ingress pool will start with a single worker", - config_pool_size => PoolSize, - pool_size => 1 - }), - 1; - {_Filter, #{}} when PoolSize == 1 -> - 1 - end. + msg => "mqtt_publisher_clean_start_false", + reason => "clean_start is set to false when using MQTT publisher action, " ++ + "which may cause unexpected behavior. " ++ + "For example, if the client ID is already subscribed to topics, " ++ + "we might receive messages that are unhanded.", + channel => ChannelId, + config => emqx_utils:redact(ChannelConfig) + }); + true -> + ok + end, + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState = emqx_bridge_mqtt_egress:config(#{remote => RemoteParams, local => LocalParams}), + NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}; +on_add_channel( + _ResourceId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex, + server := Server + } = OldState, + ChannelId, + #{hookpoints := HookPoints} = ChannelConfig +) -> + %% Add ingress channel + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState0 = #{ + hookpoints => HookPoints, + server => Server, + config_root => sources, + local => LocalParams, + remote => RemoteParams + }, + ChannelState1 = mk_ingress_config(ChannelId, ChannelState0, TopicToHandlerIndex), + ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState1), + NewInstalledChannels = maps:put(ChannelId, ChannelState1, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Conf) -> - % NOTE - % We are ignoring the user configuration here because there's currently no reliable way - % to ensure proper session recovery according to the MQTT spec. - ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, ?EGRESS, Conf)), - case mk_egress_config(Conf) of - Egress = #{} -> - start_egress(ResourceId, Egress, ClientOpts); - undefined -> - {ok, #{}} - end. +on_remove_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex + } = OldState, + ChannelId +) -> + ChannelState = maps:get(ChannelId, InstalledChannels), + case ChannelState of + #{ + config_root := sources + } -> + emqx_bridge_mqtt_ingress:unsubscribe_channel( + PoolName, ChannelState, ChannelId, TopicToHandlerIndex + ), + ok; + _ -> + ok + end, + NewInstalledChannels = maps:remove(ChannelId, InstalledChannels), + %% Update state + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Egress, ClientOpts) -> - PoolName = <>, - PoolSize = maps:get(pool_size, Egress), +on_get_channel_status( + _ResId, + ChannelId, + #{ + installed_channels := Channels + } = _State +) when is_map_key(ChannelId, Channels) -> + %% The channel should be ok as long as the MQTT client is ok + connected. + +on_get_channels(ResId) -> + emqx_bridge_v2:get_channels_for_connector(ResId). + +start_mqtt_clients(ResourceId, Conf) -> + ClientOpts = mk_client_opts(ResourceId, Conf), + start_mqtt_clients(ResourceId, Conf, ClientOpts). + +start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> + PoolName = <>, + #{ + pool_size := PoolSize + } = StartConf, Options = [ {name, PoolName}, {pool_size, PoolSize}, {client_opts, ClientOpts} ], - ok = emqx_resource:allocate_resource(ResourceId, egress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_egress, Options) of + ok = emqx_resource:allocate_resource(ResourceId, pool_name, PoolName), + case emqx_resource_pool:start(PoolName, ?MODULE, Options) of ok -> - {ok, #{ - egress_pool_name => PoolName, - egress_config => emqx_bridge_mqtt_egress:config(Egress) - }}; + {ok, #{pool_name => PoolName}}; {error, {start_pool_failed, _, Reason}} -> {error, Reason} end. -on_stop(ResourceId, _State) -> +on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", connector => ResourceId }), + %% on_stop can be called with State = undefined + StateMap = + case State of + Map when is_map(State) -> + Map; + _ -> + #{} + end, + case maps:get(topic_to_handler_index, StateMap, undefined) of + undefined -> + ok; + TopicToHandlerIndex -> + ets:delete(TopicToHandlerIndex) + end, Allocated = emqx_resource:get_allocated_resources(ResourceId), - ok = stop_ingress(Allocated), - ok = stop_egress(Allocated). - -stop_ingress(#{ingress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_ingress(#{}) -> + ok = stop_helper(Allocated), + ?tp(mqtt_connector_stopped, #{instance_id => ResourceId}), ok. -stop_egress(#{egress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_egress(#{}) -> - ok. +stop_helper(#{pool_name := PoolName}) -> + emqx_resource_pool:stop(PoolName). on_query( ResourceId, - {send_message, Msg}, - #{egress_pool_name := PoolName, egress_config := Config} + {ChannelId, Msg}, + #{pool_name := PoolName} = State ) -> - ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - handle_send_result(with_egress_client(PoolName, send, [Msg, Config])); -on_query(ResourceId, {send_message, Msg}, #{}) -> + ?TRACE( + "QUERY", + "send_msg_to_remote_node", + #{ + message => Msg, + connector => ResourceId, + channel_id => ChannelId + } + ), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + handle_send_result(with_egress_client(PoolName, send, [Msg, ChannelConfig])); +on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -187,13 +275,15 @@ on_query(ResourceId, {send_message, Msg}, #{}) -> on_query_async( ResourceId, - {send_message, Msg}, + {ChannelId, Msg}, CallbackIn, - #{egress_pool_name := PoolName, egress_config := Config} + #{pool_name := PoolName} = State ) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - Result = with_egress_client(PoolName, send_async, [Msg, Callback, Config]), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + Result = with_egress_client(PoolName, send_async, [Msg, Callback, ChannelConfig]), case Result of ok -> ok; @@ -202,7 +292,7 @@ on_query_async( {error, Reason} -> {error, classify_error(Reason)} end; -on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> +on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -251,7 +341,7 @@ classify_error(Reason) -> {unrecoverable_error, Reason}. on_get_status(_ResourceId, State) -> - Pools = maps:to_list(maps:with([ingress_pool_name, egress_pool_name], State)), + Pools = maps:to_list(maps:with([pool_name], State)), Workers = [{Pool, Worker} || {Pool, PN} <- Pools, {_Name, Worker} <- ecpool:workers(PN)], try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of Statuses -> @@ -261,12 +351,10 @@ on_get_status(_ResourceId, State) -> connecting end. -get_status({Pool, Worker}) -> +get_status({_Pool, Worker}) -> case ecpool_worker:client(Worker) of - {ok, Client} when Pool == ingress_pool_name -> + {ok, Client} -> emqx_bridge_mqtt_ingress:status(Client); - {ok, Client} when Pool == egress_pool_name -> - emqx_bridge_mqtt_egress:status(Client); {error, _} -> disconnected end. @@ -284,30 +372,19 @@ combine_status(Statuses) -> end. mk_ingress_config( - ResourceId, - #{ - ingress := Ingress = #{remote := _}, - server := Server, - hookpoint := HookPoint - } + ChannelId, + IngressChannelConfig, + TopicToHandlerIndex ) -> - Ingress#{ - server => Server, - on_message_received => {?MODULE, on_message_received, [HookPoint, ResourceId]} - }; -mk_ingress_config(ResourceId, #{ingress := #{remote := _}} = Conf) -> - error({no_hookpoint_provided, ResourceId, Conf}); -mk_ingress_config(_ResourceId, #{}) -> - undefined. - -mk_egress_config(#{egress := Egress = #{remote := _}}) -> - Egress; -mk_egress_config(#{}) -> - undefined. + HookPoints = maps:get(hookpoints, IngressChannelConfig, []), + NewConf = IngressChannelConfig#{ + on_message_received => {?MODULE, on_message_received, [HookPoints, ChannelId]}, + ingress_list => [IngressChannelConfig] + }, + emqx_bridge_mqtt_ingress:config(NewConf, ChannelId, TopicToHandlerIndex). mk_client_opts( ResourceId, - ClientScope, Config = #{ server := Server, keepalive := KeepAlive, @@ -327,14 +404,15 @@ mk_client_opts( % A load balancing server (such as haproxy) is often set up before the emqx broker server. % When the load balancing server enables mqtt connection packet inspection, % non-standard mqtt connection packets might be filtered out by LB. - bridge_mode + bridge_mode, + topic_to_handler_index ], Config ), Name = parse_id_to_name(ResourceId), mk_client_opt_password(Options#{ hosts => [HostPort], - clientid => clientid(Name, ClientScope, Config), + clientid => clientid(Name, Config), connect_timeout => 30, keepalive => ms_to_s(KeepAlive), force_ping => true, @@ -342,10 +420,8 @@ mk_client_opts( ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }). -parse_id_to_name(<>) -> - Name; parse_id_to_name(Id) -> - {_Type, Name} = emqx_bridge_resource:parse_bridge_id(Id, #{atom_name => false}), + {_Type, Name} = emqx_connector_resource:parse_connector_id(Id, #{atom_name => false}), Name. mk_client_opt_password(Options = #{password := Secret}) -> @@ -357,9 +433,82 @@ mk_client_opt_password(Options) -> ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Name, ClientScope, _Conf = #{clientid_prefix := Prefix}) when +clientid(Name, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) andalso Prefix =/= <<>> -> - emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name, ClientScope]); -clientid(Name, ClientScope, _Conf) -> - emqx_bridge_mqtt_lib:clientid_base([Name, ClientScope]). + emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name]); +clientid(Name, _Conf) -> + emqx_bridge_mqtt_lib:clientid_base([Name]). + +%% @doc Start an ingress bridge worker. +-spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(Options) -> + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ?SLOG(debug, #{ + msg => "ingress_client_starting", + options => emqx_utils:redact(Options) + }), + Name = proplists:get_value(name, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ClientOpts = proplists:get_value(client_opts, Options), + case emqtt:start_link(mk_client_opts(Name, WorkerId, ClientOpts)) of + {ok, Pid} -> + connect(Pid, Name); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "client_start_failed", + config => emqx_utils:redact(ClientOpts), + reason => Reason + }), + Error + end. + +mk_client_opts( + Name, + WorkerId, + ClientOpts = #{ + clientid := ClientId, + topic_to_handler_index := TopicToHandlerIndex + } +) -> + ClientOpts#{ + clientid := mk_clientid(WorkerId, ClientId), + msg_handler => mk_client_event_handler(Name, TopicToHandlerIndex) + }. + +mk_clientid(WorkerId, ClientId) -> + emqx_bridge_mqtt_lib:bytes23([ClientId], WorkerId). + +mk_client_event_handler(Name, TopicToHandlerIndex) -> + #{ + publish => {fun emqx_bridge_mqtt_ingress:handle_publish/3, [Name, TopicToHandlerIndex]}, + disconnected => {fun ?MODULE:handle_disconnect/1, []} + }. + +-spec connect(pid(), name()) -> + {ok, pid()} | {error, _Reason}. +connect(Pid, Name) -> + case emqtt:connect(Pid) of + {ok, _Props} -> + {ok, Pid}; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "ingress_client_connect_failed", + reason => Reason, + name => Name + }), + _ = catch emqtt:stop(Pid), + Error + end. + +handle_disconnect(_Reason) -> + ok. + +take(Key, Map0, Default) -> + case maps:take(Key, Map0) of + {Value, Map} -> + {Value, Map}; + error -> + {Default, Map0} + end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 32f9e9295..edbb40cbb 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------- %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,10 +30,15 @@ parse_server/1 ]). +-export([ + connector_examples/1 +]). + -import(emqx_schema, [mk_duration/2]). -import(hoconsc, [mk/2, ref/2]). +-define(CONNECTOR_TYPE, mqtt). -define(MQTT_HOST_OPTS, #{default_port => 1883}). namespace() -> "connector_mqtt". @@ -61,6 +66,14 @@ fields("config") -> } )} ]; +fields("config_connector") -> + emqx_connector_schema:common_fields() ++ fields("specific_connector_config"); +fields("specific_connector_config") -> + [{pool_size, fun egress_pool_size/1}] ++ + emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + fields("server_configs"); +fields(resource_opts) -> + emqx_connector_schema:resource_opts_fields(); fields("server_configs") -> [ {mode, @@ -131,6 +144,7 @@ fields("server_configs") -> fields("ingress") -> [ {pool_size, fun ingress_pool_size/1}, + %% array {remote, mk( ref(?MODULE, "ingress_remote"), @@ -144,6 +158,22 @@ fields("ingress") -> } )} ]; +fields(connector_ingress) -> + [ + {remote, + mk( + ref(?MODULE, "ingress_remote"), + #{desc => ?DESC("ingress_remote")} + )}, + {local, + mk( + ref(?MODULE, "ingress_local"), + #{ + desc => ?DESC("ingress_local"), + importance => ?IMPORTANCE_HIDDEN + } + )} + ]; fields("ingress_remote") -> [ {topic, @@ -269,7 +299,16 @@ fields("egress_remote") -> desc => ?DESC("payload") } )} - ]. + ]; +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + Fields = fields("specific_connector_config"), + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, Fields); +fields(What) -> + error({emqx_bridge_mqtt_connector_schema, missing_field_handler, What}). ingress_pool_size(desc) -> ?DESC("ingress_pool_size"); @@ -283,6 +322,8 @@ egress_pool_size(Prop) -> desc("server_configs") -> ?DESC("server_configs"); +desc("config_connector") -> + ?DESC("config_connector"); desc("ingress") -> ?DESC("ingress_desc"); desc("ingress_remote") -> @@ -295,6 +336,8 @@ desc("egress_remote") -> ?DESC("egress_remote"); desc("egress_local") -> ?DESC("egress_local"); +desc(resource_opts) -> + ?DESC(emqx_resource_schema, <<"resource_opts">>); desc(_) -> undefined. @@ -304,3 +347,6 @@ qos() -> parse_server(Str) -> #{hostname := Host, port := Port} = emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS), {Host, Port}. + +connector_examples(_Method) -> + [#{}]. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 2573cad8b..38bdd9665 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -20,33 +20,16 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). - -export([ config/1, send/3, send_async/4 ]). -%% management APIs --export([ - status/1, - info/1 -]). - --type name() :: term(). -type message() :: emqx_types:message() | map(). -type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. -type remote_message() :: #mqtt_msg{}. --type option() :: - {name, name()} - %% see `emqtt:option()` - | {client_opts, map()}. - -type egress() :: #{ local => #{ topic => emqx_types:topic() @@ -54,51 +37,6 @@ remote := emqx_bridge_mqtt_msg:msgvars() }. -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "egress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(WorkerId, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name); - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "egress_client_start_failed", - config => emqx_utils:redact(ClientOpts), - reason => Reason - }), - Error - end. - -mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}. - -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -connect(Pid, Name) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "egress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error - end. - -%% - -spec config(map()) -> egress(). config(#{remote := RC = #{}} = Conf) -> @@ -137,25 +75,3 @@ to_remote_msg(Msg = #{}, Remote) -> props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }. - -%% - --spec info(pid()) -> - [{atom(), term()}]. -info(Pid) -> - emqtt:info(Pid). - --spec status(pid()) -> - emqx_resource:resource_status(). -status(Pid) -> - try - case proplists:get_value(socket, info(Pid)) of - Socket when Socket /= undefined -> - connected; - undefined -> - connecting - end - catch - exit:{noproc, _} -> - disconnected - end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index a051ffbd8..369238ecf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -17,129 +17,190 @@ -module(emqx_bridge_mqtt_ingress). -include_lib("emqx/include/logger.hrl"). - --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). +-include_lib("emqx/include/emqx_mqtt.hrl"). %% management APIs -export([ status/1, - info/1 + info/1, + subscribe_channel/2, + unsubscribe_channel/4, + config/3 ]). --export([handle_publish/5]). --export([handle_disconnect/1]). +-export([handle_publish/3]). --type name() :: term(). +subscribe_channel(PoolName, ChannelConfig) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + Results = [ + subscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize) + || {{Name, Idx}, Pid} <- Workers + ], + case proplists:get_value(error, Results, ok) of + ok -> + ok; + Error -> + Error + end. --type option() :: - {name, name()} - | {ingress, map()} - %% see `emqtt:option()` - | {client_opts, map()}. +subscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize); + {error, Reason} -> + error({client_not_found, Reason}) + end. --type ingress() :: #{ - server := string(), - remote := #{ - topic := emqx_types:topic(), - qos => emqx_types:qos() - }, - local := emqx_bridge_mqtt_msg:msgvars(), - on_message_received := {module(), atom(), [term()]} -}. - -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "ingress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - Ingress = config(proplists:get_value(ingress, Options), Name), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(Name, WorkerId, Ingress, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name, Ingress); +subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize) -> + IngressList = maps:get(ingress_list, Ingress, []), + SubscribeResults = subscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name + ), + %% Find error if any using proplists:get_value/2 + case proplists:get_value(error, SubscribeResults, ok) of + ok -> + ok; {error, Reason} = Error -> ?SLOG(error, #{ - msg => "client_start_failed", - config => emqx_utils:redact(ClientOpts), + msg => "ingress_client_subscribe_failed", + ingress => Ingress, + name => Name, reason => Reason }), Error end. -mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{ - clientid := mk_clientid(WorkerId, ClientId), - msg_handler => mk_client_event_handler(Name, Ingress) - }. +subscribe_remote_topics(Pid, IngressList, WorkerIdx, PoolSize, Name) -> + [subscribe_remote_topic(Pid, Ingress, WorkerIdx, PoolSize, Name) || Ingress <- IngressList]. -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -mk_client_event_handler(Name, Ingress = #{}) -> - IngressVars = maps:with([server], Ingress), - OnMessage = maps:get(on_message_received, Ingress, undefined), - LocalPublish = - case Ingress of - #{local := Local = #{topic := _}} -> - Local; - #{} -> - undefined - end, - #{ - publish => {fun ?MODULE:handle_publish/5, [Name, OnMessage, LocalPublish, IngressVars]}, - disconnected => {fun ?MODULE:handle_disconnect/1, []} - }. - --spec connect(pid(), name(), ingress()) -> - {ok, pid()} | {error, _Reason}. -connect(Pid, Name, Ingress) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - case subscribe_remote_topic(Pid, Ingress) of - {ok, _, _RCs} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "ingress_client_subscribe_failed", - ingress => Ingress, - name => Name, - reason => Reason - }), - _ = catch emqtt:stop(Pid), - Error - end; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "ingress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error +subscribe_remote_topic( + Pid, #{remote := #{topic := RemoteTopic, qos := QoS}} = _Remote, WorkerIdx, PoolSize, Name +) -> + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _LogWarn = true) of + true -> + emqtt:subscribe(Pid, RemoteTopic, QoS); + false -> + ok end. -subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> - emqtt:subscribe(Pid, RemoteTopic, QoS). +should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, LogWarn) -> + IsFirstWorker = WorkerIdx == 1, + case emqx_topic:parse(RemoteTopic) of + {#share{} = _Filter, _SubOpts} -> + % NOTE: this is shared subscription, many workers may subscribe + true; + {_Filter, #{}} when PoolSize > 1, IsFirstWorker, LogWarn -> + % NOTE: this is regular subscription, only one worker should subscribe + ?SLOG(warning, #{ + msg => "mqtt_pool_size_ignored", + connector => Name, + reason => + "Remote topic filter is not a shared subscription, " + "only a single connection will be used from the connection pool", + config_pool_size => PoolSize, + pool_size => PoolSize + }), + IsFirstWorker; + {_Filter, #{}} -> + % NOTE: this is regular subscription, only one worker should subscribe + IsFirstWorker + end. -%% +unsubscribe_channel(PoolName, ChannelConfig, ChannelId, TopicToHandlerIndex) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + _ = [ + unsubscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize, ChannelId, TopicToHandlerIndex) + || {{Name, Idx}, Pid} <- Workers + ], + ok. --spec config(map(), name()) -> - ingress(). -config(#{remote := RC, local := LC} = Conf, BridgeName) -> - Conf#{ - remote => parse_remote(RC, BridgeName), - local => emqx_bridge_mqtt_msg:parse(LC) - }. +unsubscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex + ); + {error, Reason} -> + error({client_not_found, Reason}) + end. -parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> +unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex +) -> + IngressList = maps:get(ingress_list, Ingress, []), + unsubscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ). + +unsubscribe_remote_topics( + Pid, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex +) -> + [ + unsubscribe_remote_topic( + Pid, Ingress, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ) + || Ingress <- IngressList + ]. + +unsubscribe_remote_topic( + Pid, + #{remote := #{topic := RemoteTopic}} = _Remote, + WorkerIdx, + PoolSize, + Name, + ChannelId, + TopicToHandlerIndex +) -> + emqx_topic_index:delete(RemoteTopic, ChannelId, TopicToHandlerIndex), + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _NoWarn = false) of + true -> + case emqtt:unsubscribe(Pid, RemoteTopic) of + {ok, _Properties, _ReasonCodes} -> + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "unsubscribe_mqtt_topic_failed", + channel_id => Name, + reason => Reason + }), + ok + end; + false -> + ok + end. + +config(#{ingress_list := IngressList} = Conf, Name, TopicToHandlerIndex) -> + NewIngressList = [ + fix_remote_config(Ingress, Name, TopicToHandlerIndex, Conf) + || Ingress <- IngressList + ], + Conf#{ingress_list => NewIngressList}. + +fix_remote_config(#{remote := RC}, BridgeName, TopicToHandlerIndex, Conf) -> + FixedConf0 = Conf#{ + remote => parse_remote(RC, BridgeName) + }, + FixedConf = emqx_utils_maps:update_if_present( + local, fun emqx_bridge_mqtt_msg:parse/1, FixedConf0 + ), + insert_to_topic_to_handler_index(FixedConf, TopicToHandlerIndex, BridgeName), + FixedConf. + +insert_to_topic_to_handler_index( + #{remote := #{topic := Topic}} = Conf, TopicToHandlerIndex, BridgeName +) -> + TopicPattern = + case emqx_topic:parse(Topic) of + {#share{group = _Group, topic = TP}, _} -> + TP; + _ -> + Topic + end, + emqx_topic_index:insert(TopicPattern, BridgeName, Conf, TopicToHandlerIndex). + +parse_remote(#{qos := QoSIn} = Remote, BridgeName) -> QoS = downgrade_ingress_qos(QoSIn), case QoS of QoSIn -> @@ -152,7 +213,7 @@ parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> name => BridgeName }) end, - Conf#{qos => QoS}. + Remote#{qos => QoS}. downgrade_ingress_qos(2) -> 1; @@ -183,17 +244,39 @@ status(Pid) -> %% -handle_publish(#{properties := Props} = MsgIn, Name, OnMessage, LocalPublish, IngressVars) -> - Msg = import_msg(MsgIn, IngressVars), +handle_publish( + #{properties := Props, topic := Topic} = MsgIn, + Name, + TopicToHandlerIndex +) -> ?SLOG(debug, #{ msg => "ingress_publish_local", - message => Msg, + message => MsgIn, name => Name }), - maybe_on_message_received(Msg, OnMessage), - maybe_publish_local(Msg, LocalPublish, Props). + Matches = emqx_topic_index:matches(Topic, TopicToHandlerIndex, []), + lists:foreach( + fun(Match) -> + handle_match(TopicToHandlerIndex, Match, MsgIn, Name, Props) + end, + Matches + ), + ok. -handle_disconnect(_Reason) -> +handle_match( + TopicToHandlerIndex, + Match, + MsgIn, + _Name, + Props +) -> + [ChannelConfig] = emqx_topic_index:get_record(Match, TopicToHandlerIndex), + #{on_message_received := OnMessage} = ChannelConfig, + Msg = import_msg(MsgIn, ChannelConfig), + + maybe_on_message_received(Msg, OnMessage), + LocalPublish = maps:get(local, ChannelConfig, undefined), + _ = maybe_publish_local(Msg, LocalPublish, Props), ok. maybe_on_message_received(Msg, {Mod, Func, Args}) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl new file mode 100644 index 000000000..407a25118 --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -0,0 +1,256 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_pubsub_action_info). + +-behaviour(emqx_action_info). + +-export([ + bridge_v1_type_name/0, + action_type_name/0, + connector_type_name/0, + schema_module/0, + bridge_v1_config_to_connector_config/1, + bridge_v1_config_to_action_config/2, + connector_action_config_to_bridge_v1_config/2, + is_source/0 +]). + +bridge_v1_type_name() -> mqtt. + +action_type_name() -> mqtt. + +connector_type_name() -> mqtt. + +schema_module() -> emqx_bridge_mqtt_pubsub_schema. + +is_source() -> true. + +bridge_v1_config_to_connector_config(Config) -> + %% Transform the egress part to mqtt_publisher connector config + SimplifiedConfig = check_and_simplify_bridge_v1_config(Config), + ConnectorConfigMap = make_connector_config_from_bridge_v1_config(SimplifiedConfig), + {mqtt, ConnectorConfigMap}. + +make_connector_config_from_bridge_v1_config(Config) -> + ConnectorConfigSchema = emqx_bridge_mqtt_connector_schema:fields("config_connector"), + ConnectorTopFields = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ConnectorConfigSchema + ], + ConnectorConfigMap = maps:with(ConnectorTopFields, Config), + ResourceOptsMap = maps:get(<<"resource_opts">>, ConnectorConfigMap, #{}), + ResourceOptsMap2 = emqx_connector_schema:project_to_connector_resource_opts(ResourceOptsMap), + ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), + IngressMap0 = maps:get(<<"ingress">>, Config, #{}), + EgressMap = maps:get(<<"egress">>, Config, #{}), + %% Move pool_size to the top level + PoolSizeIngress = maps:get(<<"pool_size">>, IngressMap0, undefined), + PoolSize = + case PoolSizeIngress of + undefined -> + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + maps:get(<<"pool_size">>, EgressMap, DefaultPoolSize); + _ -> + PoolSizeIngress + end, + %% Remove ingress part from the config + ConnectorConfigMap3 = maps:remove(<<"ingress">>, ConnectorConfigMap2), + %% Remove egress part from the config + ConnectorConfigMap4 = maps:remove(<<"egress">>, ConnectorConfigMap3), + ConnectorConfigMap5 = maps:put(<<"pool_size">>, PoolSize, ConnectorConfigMap4), + ConnectorConfigMap5. + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + SimplifiedConfig = check_and_simplify_bridge_v1_config(BridgeV1Config), + bridge_v1_config_to_action_config_helper( + SimplifiedConfig, ConnectorName + ). + +bridge_v1_config_to_action_config_helper( + #{ + <<"egress">> := EgressMap0 + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_publisher_action"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), + LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), + EgressMap1 = maps:without([<<"local">>, <<"pool_size">>], EgressMap0), + LocalParams = maps:get(<<"local">>, EgressMap0, #{}), + EgressMap2 = emqx_utils_maps:unindent(<<"remote">>, EgressMap1), + EgressMap = maps:put(<<"local">>, LocalParams, EgressMap2), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, EgressMap, ConfigMap1), + ConfigMap3 = + case LocalTopic of + undefined -> + ConfigMap2; + _ -> + maps:put(<<"local_topic">>, LocalTopic, ConfigMap2) + end, + {action, mqtt, ConfigMap3}; +bridge_v1_config_to_action_config_helper( + #{ + <<"ingress">> := IngressMap0 + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(source_resource_opts), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + IngressMap1 = maps:without([<<"pool_size">>, <<"local">>], IngressMap0), + LocalParams = maps:get(<<"local">>, IngressMap0, #{}), + IngressMap2 = emqx_utils_maps:unindent(<<"remote">>, IngressMap1), + IngressMap = maps:put(<<"local">>, LocalParams, IngressMap2), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, IngressMap, ConfigMap1), + {source, mqtt, ConfigMap2}; +bridge_v1_config_to_action_config_helper( + _Config, + _ConnectorName +) -> + error({incompatible_bridge_v1, no_matching_action_or_source}). + +general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields +) -> + ShemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- SchemaFields + ], + ActionConfig0 = maps:with(ShemaFieldsNames, Config), + ResourceOptsSchemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ResourceOptsSchemaFields + ], + ResourceOptsMap = maps:get(<<"resource_opts">>, ActionConfig0, #{}), + ResourceOptsMap2 = maps:with(ResourceOptsSchemaFieldsNames, ResourceOptsMap), + %% Only put resource_opts if the original config has it + ActionConfig1 = + case maps:is_key(<<"resource_opts">>, ActionConfig0) of + true -> + maps:put(<<"resource_opts">>, ResourceOptsMap2, ActionConfig0); + false -> + ActionConfig0 + end, + ActionConfig2 = maps:put(<<"connector">>, ConnectorName, ActionConfig1), + ActionConfig2. + +check_and_simplify_bridge_v1_config( + #{ + <<"egress">> := EgressMap + } = Config +) when map_size(EgressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"egress">>, Config)); +check_and_simplify_bridge_v1_config( + #{ + <<"ingress">> := IngressMap + } = Config +) when map_size(IngressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"ingress">>, Config)); +check_and_simplify_bridge_v1_config(#{ + <<"egress">> := _EGressMap, + <<"ingress">> := _InGressMap +}) -> + %% We should crash beacuse we don't support upgrading when ingress and egress exist at the same time + error( + {unsupported_config, << + "Upgrade not supported when ingress and egress exist in the " + "same MQTT bridge. Please divide the egress and ingress part " + "to separate bridges in the configuration." + >>} + ); +check_and_simplify_bridge_v1_config(SimplifiedConfig) -> + SimplifiedConfig. + +connector_action_config_to_bridge_v1_config( + ConnectorConfig, ActionConfig +) -> + Params0 = maps:get(<<"parameters">>, ActionConfig, #{}), + ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), + ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), + ResourceOpts0 = maps:merge(ResourceOptsConnector, ResourceOptsAction), + V1ResourceOptsFields = + lists:map( + fun({Field, _}) -> atom_to_binary(Field) end, + emqx_bridge_mqtt_schema:fields("creation_opts") + ), + ResourceOpts = maps:with(V1ResourceOptsFields, ResourceOpts0), + %% Check the direction of the action + Direction = + case is_map_key(<<"retain">>, Params0) of + %% Only source has retain + true -> + <<"publisher">>; + false -> + <<"subscriber">> + end, + Params1 = maps:remove(<<"direction">>, Params0), + Params = maps:remove(<<"local">>, Params1), + %% hidden; for backwards compatibility + LocalParams = maps:get(<<"local">>, Params1, #{}), + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + PoolSize = maps:get(<<"pool_size">>, ConnectorConfig, DefaultPoolSize), + ConnectorConfig2 = maps:remove(<<"pool_size">>, ConnectorConfig), + LocalTopic = maps:get(<<"local_topic">>, ActionConfig, undefined), + BridgeV1Conf0 = + case {Direction, LocalTopic} of + {<<"publisher">>, undefined} -> + #{ + <<"egress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + }; + {<<"publisher">>, LocalT} -> + #{ + <<"egress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => + maps:merge( + LocalParams, + #{<<"topic">> => LocalT} + ) + } + }; + {<<"subscriber">>, _} -> + #{ + <<"ingress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + } + end, + BridgeV1Conf1 = maps:merge(BridgeV1Conf0, ConnectorConfig2), + BridgeV1Conf2 = BridgeV1Conf1#{ + <<"resource_opts">> => ResourceOpts + }, + BridgeV1Conf2. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl new file mode 100644 index 000000000..05b2d6d3a --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -0,0 +1,206 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_pubsub_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/2]). + +-export([roots/0, fields/1, desc/1, namespace/0]). + +-export([ + bridge_v2_examples/1, + source_examples/1, + conn_bridge_examples/1 +]). + +-define(ACTION_TYPE, mqtt). +-define(SOURCE_TYPE, mqtt). + +%%====================================================================================== +%% Hocon Schema Definitions +namespace() -> "bridge_mqtt_publisher". + +roots() -> []. + +fields(action) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_publisher_action")), + #{ + desc => <<"MQTT Publisher Action Config">>, + required => false + } + )}; +fields("mqtt_publisher_action") -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + hoconsc:ref(?MODULE, action_parameters), + #{ + required => true, + desc => ?DESC("action_parameters") + } + ) + ); +fields(action_parameters) -> + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "egress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("egress_remote") + ]; +fields(source) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_subscriber_source")), + #{ + desc => <<"MQTT Subscriber Source Config">>, + required => false + } + )}; +fields("mqtt_subscriber_source") -> + emqx_bridge_v2_schema:make_consumer_action_schema( + mk( + ref(?MODULE, ingress_parameters), + #{ + required => true, + desc => ?DESC("source_parameters") + } + ) + ); +fields(ingress_parameters) -> + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "ingress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("ingress_remote") + ]; +fields(action_resource_opts) -> + UnsupportedOpts = [enable_batch, batch_size, batch_time], + lists:filter( + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, + emqx_bridge_v2_schema:action_resource_opts_fields() + ); +fields(source_resource_opts) -> + emqx_bridge_v2_schema:source_resource_opts_fields(); +fields(Field) when + Field == "get_bridge_v2"; + Field == "post_bridge_v2"; + Field == "put_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION_TYPE, fields("mqtt_publisher_action")); +fields(Field) when + Field == "get_source"; + Field == "post_source"; + Field == "put_source" +-> + emqx_bridge_v2_schema:api_fields(Field, ?SOURCE_TYPE, fields("mqtt_subscriber_source")); +fields(What) -> + error({emqx_bridge_mqtt_pubsub_schema, missing_field_handler, What}). +%% v2: api schema +%% The parameter equls to +%% `get_bridge_v2`, `post_bridge_v2`, `put_bridge_v2` from emqx_bridge_v2_schema:api_schema/1 +%% `get_connector`, `post_connector`, `put_connector` from emqx_connector_schema:api_schema/1 +%%-------------------------------------------------------------------- +%% v1/v2 + +desc("config") -> + ?DESC("desc_config"); +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc(source_resource_opts) -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc(action_parameters) -> + ?DESC(action_parameters); +desc(ingress_parameters) -> + ?DESC(ingress_parameters); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; +desc("http_action") -> + ?DESC("desc_config"); +desc("parameters_opts") -> + ?DESC("config_parameters_opts"); +desc("mqtt_publisher_action") -> + ?DESC("mqtt_publisher_action"); +desc("mqtt_subscriber_source") -> + ?DESC("mqtt_subscriber_source"); +desc(_) -> + undefined. + +bridge_v2_examples(Method) -> + [ + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, + _ActionType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2, + retain => false, + payload => <<"${.payload}">> + } + } + ) + } + } + ]. + +source_examples(Method) -> + [ + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Subscriber Source">>, + value => emqx_bridge_v2_schema:source_values( + Method, + _SourceType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2 + } + } + ) + } + } + ]. + +conn_bridge_examples(Method) -> + [ + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_api:mqtt_v1_example(Method) + } + } + ]. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl index a312dfaa9..37d0b4f03 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -13,6 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- + -module(emqx_bridge_mqtt_schema). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6d1ff0915..807fba3c9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -238,7 +238,8 @@ t_conf_bridge_authn_passfile(Config) -> post, uri(["bridges"]), ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ - <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + <<"name">> => <<"t_conf_bridge_authn_no_passfile">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} } ), ?assertMatch({match, _}, re:run(Reason, <<"failed_to_read_secret_file">>)). @@ -397,32 +398,28 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), ok. -t_mqtt_egress_bridge_ignores_clean_start(_) -> +t_mqtt_egress_bridge_warns_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), - BridgeID = create_bridge( - ?SERVER_CONF#{ - <<"name">> => BridgeName, - <<"egress">> => ?EGRESS_CONF, - <<"clean_start">> => false - } - ), + Action = fun() -> + BridgeID = create_bridge( + ?SERVER_CONF#{ + <<"name">> => BridgeName, + <<"egress">> => ?EGRESS_CONF, + <<"clean_start">> => false + } + ), - ResourceID = emqx_bridge_resource:resource_id(BridgeID), - {ok, _Group, #{state := #{egress_pool_name := EgressPoolName}}} = - emqx_resource_manager:lookup_cached(ResourceID), - ClientInfo = ecpool:pick_and_do( - EgressPoolName, - {emqx_bridge_mqtt_egress, info, []}, - no_handover - ), - ?assertMatch( - #{clean_start := true}, - maps:from_list(ClientInfo) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + ok + end, + {ok, {ok, _}} = + ?wait_async_action( + Action(), + #{?snk_kind := mqtt_clean_start_egress_action_warning}, + 10000 + ), ok. t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> @@ -567,17 +564,17 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> t_egress_short_clientid(_Config) -> %% Name is short, expect the actual client ID in use is hashed from - %% E: - Name = "abc01234", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc01234">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = iolist_to_binary([BaseId, $:, "1"]), test_egress_clientid(Name, ExpectedClientId). t_egress_long_clientid(_Config) -> %% Expect the actual client ID in use is hashed from - %% E: - Name = "abc01234567890123456789", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc012345678901234567890">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = emqx_bridge_mqtt_lib:bytes23(BaseId, 1), test_egress_clientid(Name, ExpectedClientId). @@ -1052,7 +1049,8 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - emqx_utils_json:decode(Bridge) + emqx_utils_json:decode(Bridge), + #{expected_type => Type, expected_name => Name} ), emqx_bridge_resource:bridge_id(Type, Name). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl new file mode 100644 index 000000000..3e5471d55 --- /dev/null +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -0,0 +1,244 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_v2_subscriber_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, Api} = emqx_common_test_http:create_default_app(), + [ + {apps, Apps}, + {api, Api} + | Config + ]. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), + ok. + +init_per_testcase(TestCase, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + Name = iolist_to_binary([atom_to_binary(TestCase), UniqueNum]), + ConnectorConfig = connector_config(), + SourceConfig = source_config(#{connector => Name}), + [ + {bridge_kind, source}, + {source_type, mqtt}, + {source_name, Name}, + {source_config, SourceConfig}, + {connector_type, mqtt}, + {connector_name, Name}, + {connector_config, ConnectorConfig} + | Config + ]. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +connector_config() -> + %% !!!!!!!!!!!! FIXME!!!!!! add more fields ("server_configs") + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + CommonConfig = + #{ + <<"enable">> => true, + <<"connector">> => <<"please override">>, + <<"parameters">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"resume_interval">> => <<"15s">> + } + }, + maps:merge(CommonConfig, Overrides). + +replace(Key, Value, Proplist) -> + lists:keyreplace(Key, 1, Proplist, {Key, Value}). + +bridge_id(Config) -> + Type = ?config(source_type, Config), + Name = ?config(source_name, Config), + emqx_bridge_resource:bridge_id(Type, Name). + +hookpoint(Config) -> + BridgeId = bridge_id(Config), + emqx_bridge_resource:bridge_hookpoint(BridgeId). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_create_via_http(Config) -> + ConnectorName = ?config(connector_name, Config), + ok = emqx_bridge_v2_testlib:t_create_via_http(Config), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"enable">> := true, + <<"status">> := <<"connected">> + } + ]}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + ?assertMatch( + {ok, {{_, 200, _}, _, [#{<<"enable">> := true}]}}, + emqx_bridge_v2_testlib:list_connectors_http_api() + ), + + NewSourceName = <<"my_other_source">>, + {ok, {{_, 201, _}, _, _}} = + emqx_bridge_v2_testlib:create_kind_api( + replace(source_name, NewSourceName, Config) + ), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{<<"connector">> := ConnectorName}, + #{<<"connector">> := ConnectorName} + ]}}, + emqx_bridge_v2_testlib:list_sources_http_api() + ), + ?assertMatch( + {ok, {{_, 200, _}, _, []}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + ok. + +t_start_stop(Config) -> + ok = emqx_bridge_v2_testlib:t_start_stop(Config, mqtt_connector_stopped), + ok. + +t_receive_via_rule(Config) -> + SourceConfig = ?config(source_config, Config), + ?check_trace( + begin + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_connector_api(Config), + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_kind_api(Config), + Hookpoint = hookpoint(Config), + RepublishTopic = <<"rep/t">>, + RemoteTopic = emqx_utils_maps:deep_get( + [<<"parameters">>, <<"topic">>], + SourceConfig + ), + RuleOpts = #{ + sql => <<"select * from \"", Hookpoint/binary, "\"">>, + actions => [ + %% #{function => console}, + #{ + function => republish, + args => #{ + topic => RepublishTopic, + payload => <<"${.}">>, + qos => 0, + retain => false, + user_properties => <<"${.pub_props.'User-Property'}">> + } + } + ] + }, + {ok, {{_, 201, _}, _, #{<<"id">> := RuleId}}} = + emqx_bridge_v2_testlib:create_rule_api(RuleOpts), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), + {ok, Client} = emqtt:start_link([{proto_ver, v5}]), + {ok, _} = emqtt:connect(Client), + {ok, _, [?RC_GRANTED_QOS_0]} = emqtt:subscribe(Client, RepublishTopic), + ok = emqtt:publish( + Client, + RemoteTopic, + #{'User-Property' => [{<<"key">>, <<"value">>}]}, + <<"mypayload">>, + _Opts = [] + ), + {publish, Msg} = + ?assertReceive( + {publish, #{ + topic := RepublishTopic, + retain := false, + qos := 0, + properties := #{'User-Property' := [{<<"key">>, <<"value">>}]} + }} + ), + Payload = emqx_utils_json:decode(maps:get(payload, Msg), [return_maps]), + ?assertMatch( + #{ + <<"event">> := Hookpoint, + <<"payload">> := <<"mypayload">> + }, + Payload + ), + emqtt:stop(Client), + ok + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("action_references_nonexistent_bridges", Trace)), + ok + end + ), + ok. diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 0cefd6af4..a74e80fa1 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -166,11 +166,6 @@ common_init(Config0) -> #{work_dir => emqx_cth_suite:work_dir(Config0)} ), {ok, _Api} = emqx_common_test_http:create_default_app(), - - %% ok = emqx_common_test_helpers:start_apps([emqx, emqx_postgresql, emqx_conf, emqx_bridge]), - %% _ = emqx_bridge_enterprise:module_info(), - %% emqx_mgmt_api_test_util:init_suite(), - % Connect to pgsql directly and create the table connect_and_create_table(Config0), {Name, PGConf} = pgsql_config(BridgeType, Config0), diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index 9373fe8bd..adda91f37 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -76,7 +76,7 @@ fields(redis_action) -> [ResOpts] = emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts), lists:keyreplace(resource_opts, 1, Schema, ResOpts); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{desc => ?DESC(batch_size)}}, {batch_time, #{desc => ?DESC(batch_time)}} ]); diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 83d387cd7..aae913001 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -326,7 +326,7 @@ schema("/connectors_probe") -> create_connector(ConnectorType, ConnectorName, Conf) end; '/connectors'(get, _Params) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), NodeReplies = emqx_connector_proto_v1:list_connectors_on_nodes(Nodes), case is_ok(NodeReplies) of {ok, NodeConnectors} -> @@ -674,7 +674,10 @@ unpack_connector_conf(Type, PackedConf) -> RawConf. format_action(ActionId) -> - element(2, emqx_bridge_v2:parse_id(ActionId)). + case emqx_bridge_v2:parse_id(ActionId) of + #{name := Name} -> + Name + end. is_ok(ok) -> ok; diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index a58a1ef3d..f85109080 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -51,11 +51,11 @@ -export([parse_url/1]). --callback connector_config(ParsedConfig) -> +-callback connector_config(ParsedConfig, Context) -> ParsedConfig when - ParsedConfig :: #{atom() => any()}. --optional_callbacks([connector_config/1]). + ParsedConfig :: #{atom() => any()}, Context :: #{atom() => any()}. +-optional_callbacks([connector_config/2]). -if(?EMQX_RELEASE_EDITION == ee). connector_to_resource_type(ConnectorType) -> @@ -81,6 +81,10 @@ connector_impl_module(_ConnectorType) -> connector_to_resource_type_ce(http) -> emqx_bridge_http_connector; +connector_to_resource_type_ce(mqtt) -> + emqx_bridge_mqtt_connector; +% connector_to_resource_type_ce(mqtt_subscriber) -> +% emqx_bridge_mqtt_subscriber_connector; connector_to_resource_type_ce(ConnectorType) -> error({no_bridge_v2, ConnectorType}). @@ -276,6 +280,12 @@ remove(Type, Name, _Conf, _Opts) -> emqx_resource:remove_local(resource_id(Type, Name)). %% convert connector configs to what the connector modules want +parse_confs( + <<"mqtt">> = Type, + Name, + Conf +) -> + insert_hookpoints(Type, Name, Conf); parse_confs( <<"http">>, _Name, @@ -307,6 +317,13 @@ parse_confs( parse_confs(ConnectorType, Name, Config) -> connector_config(ConnectorType, Name, Config). +insert_hookpoints(Type, Name, Conf) -> + BId = emqx_bridge_resource:bridge_id(Type, Name), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + ConnectorHookpoint = connector_hookpoint(BId), + HookPoints = [BridgeHookpoint, ConnectorHookpoint], + Conf#{hookpoints => HookPoints}. + connector_config(ConnectorType, Name, Config) -> Mod = connector_impl_module(ConnectorType), case erlang:function_exported(Mod, connector_config, 2) of diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index ad28d0251..615b89230 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -90,7 +90,9 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector") + api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector"), + % api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt_subscriber">>, Method ++ "_connector"), + api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector") ]. api_ref(Module, Type, Method) -> @@ -110,10 +112,11 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - [emqx_bridge_http_schema] ++ emqx_connector_ee_schema:schema_modules(). + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ + emqx_connector_ee_schema:schema_modules(). -else. schema_modules() -> - [emqx_bridge_http_schema]. + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema]. -endif. %% @doc Return old bridge(v1) and/or connector(v2) type @@ -136,6 +139,8 @@ connector_type_to_bridge_types(influxdb) -> [influxdb, influxdb_api_v1, influxdb_api_v2]; connector_type_to_bridge_types(mysql) -> [mysql]; +connector_type_to_bridge_types(mqtt) -> + [mqtt]; connector_type_to_bridge_types(pgsql) -> [pgsql]; connector_type_to_bridge_types(redis) -> @@ -151,7 +156,8 @@ connector_type_to_bridge_types(iotdb) -> connector_type_to_bridge_types(elasticsearch) -> [elasticsearch]. -actions_config_name() -> <<"actions">>. +actions_config_name(action) -> <<"actions">>; +actions_config_name(source) -> <<"sources">>. has_connector_field(BridgeConf, ConnectorFields) -> lists:any( @@ -185,40 +191,52 @@ bridge_configs_to_transform( end. split_bridge_to_connector_and_action( - {ConnectorsMap, {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig}} + { + {ConnectorsMap, OrgConnectorType}, + {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig} + } ) -> - ConnectorMap = + {ConnectorMap, ConnectorType} = case emqx_action_info:has_custom_bridge_v1_config_to_connector_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_connector_config( - BridgeType, BridgeV1Conf - ); + case + emqx_action_info:bridge_v1_config_to_connector_config( + BridgeType, BridgeV1Conf + ) + of + {ConType, ConMap} -> + {ConMap, ConType}; + ConMap -> + {ConMap, OrgConnectorType} + end; false -> %% We do an automatic transformation to get the connector config %% if the callback is not defined. %% Get connector fields from bridge config - lists:foldl( - fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> - ConnectorFieldNameBin = to_bin(ConnectorFieldName), - case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of - true -> - PrevFieldConfig = - maybe_project_to_connector_resource_opts( + NewCConMap = + lists:foldl( + fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> + ConnectorFieldNameBin = to_bin(ConnectorFieldName), + case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of + true -> + PrevFieldConfig = + maybe_project_to_connector_resource_opts( + ConnectorFieldNameBin, + maps:get(ConnectorFieldNameBin, BridgeV1Conf) + ), + maps:put( ConnectorFieldNameBin, - maps:get(ConnectorFieldNameBin, BridgeV1Conf) - ), - maps:put( - ConnectorFieldNameBin, - PrevFieldConfig, + PrevFieldConfig, + ToTransformSoFar + ); + false -> ToTransformSoFar - ); - false -> - ToTransformSoFar - end - end, - #{}, - ConnectorFields - ) + end + end, + #{}, + ConnectorFields + ), + {NewCConMap, OrgConnectorType} end, %% Generate a connector name, if needed. Avoid doing so if there was a previous config. ConnectorName = @@ -226,18 +244,29 @@ split_bridge_to_connector_and_action( #{<<"connector">> := ConnectorName0} -> ConnectorName0; _ -> generate_connector_name(ConnectorsMap, BridgeName, 0) end, - ActionMap = + OrgActionType = emqx_action_info:bridge_v1_type_to_action_type(BridgeType), + {ActionMap, ActionType, ActionOrSource} = case emqx_action_info:has_custom_bridge_v1_config_to_action_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_action_config( - BridgeType, BridgeV1Conf, ConnectorName - ); + case + emqx_action_info:bridge_v1_config_to_action_config( + BridgeType, BridgeV1Conf, ConnectorName + ) + of + {ActionOrSource0, ActionType0, ActionMap0} -> + {ActionMap0, ActionType0, ActionOrSource0}; + ActionMap0 -> + {ActionMap0, OrgActionType, action} + end; false -> - transform_bridge_v1_config_to_action_config( - BridgeV1Conf, ConnectorName, ConnectorFields - ) + ActionMap0 = + transform_bridge_v1_config_to_action_config( + BridgeV1Conf, ConnectorName, ConnectorFields + ), + {ActionMap0, OrgActionType, action} end, - {BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}. + {BridgeType, BridgeName, ActionMap, ActionType, ActionOrSource, ConnectorName, ConnectorMap, + ConnectorType}. maybe_project_to_connector_resource_opts(<<"resource_opts">>, OldResourceOpts) -> project_to_connector_resource_opts(OldResourceOpts); @@ -307,9 +336,9 @@ generate_connector_name(ConnectorsMap, BridgeName, Attempt) -> ConnectorNameList = case Attempt of 0 -> - io_lib:format("connector_~s", [BridgeName]); + io_lib:format("~s", [BridgeName]); _ -> - io_lib:format("connector_~s_~p", [BridgeName, Attempt + 1]) + io_lib:format("~s_~p", [BridgeName, Attempt + 1]) end, ConnectorName = iolist_to_binary(ConnectorNameList), case maps:is_key(ConnectorName, ConnectorsMap) of @@ -340,7 +369,10 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), ConnectorsWithTypeMap = maps:get(to_bin(ConnectorType), ConnectorsConfMap, #{}), BridgeConfigsToTransformWithConnectorConf = lists:zip( - lists:duplicate(length(BridgeConfigsToTransform), ConnectorsWithTypeMap), + lists:duplicate( + length(BridgeConfigsToTransform), + {ConnectorsWithTypeMap, ConnectorType} + ), BridgeConfigsToTransform ), ActionConnectorTuples = lists:map( @@ -349,10 +381,14 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), %% Add connectors and actions and remove bridges lists:foldl( - fun({BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}, RawConfigSoFar) -> + fun( + {BridgeType, BridgeName, ActionMap, NewActionType, ActionOrSource, ConnectorName, + ConnectorMap, NewConnectorType}, + RawConfigSoFar + ) -> %% Add connector RawConfigSoFar1 = emqx_utils_maps:deep_put( - [<<"connectors">>, to_bin(ConnectorType), ConnectorName], + [<<"connectors">>, to_bin(NewConnectorType), ConnectorName], RawConfigSoFar, ConnectorMap ), @@ -362,12 +398,21 @@ transform_old_style_bridges_to_connector_and_actions_of_type( RawConfigSoFar1 ), %% Add action - ActionType = emqx_action_info:bridge_v1_type_to_action_type(to_bin(BridgeType)), - RawConfigSoFar3 = emqx_utils_maps:deep_put( - [actions_config_name(), to_bin(ActionType), BridgeName], - RawConfigSoFar2, - ActionMap - ), + RawConfigSoFar3 = + case ActionMap of + none -> + RawConfigSoFar2; + _ -> + emqx_utils_maps:deep_put( + [ + actions_config_name(ActionOrSource), + to_bin(NewActionType), + BridgeName + ], + RawConfigSoFar2, + ActionMap + ) + end, RawConfigSoFar3 end, RawConfig, @@ -454,7 +499,23 @@ fields(connectors) -> desc => <<"HTTP Connector Config">>, required => false } + )}, + {mqtt, + mk( + hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + #{ + desc => <<"MQTT Publisher Connector Config">>, + required => false + } )} + % {mqtt_subscriber, + % mk( + % hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + % #{ + % desc => <<"MQTT Subscriber Connector Config">>, + % required => false + % } + % )} ] ++ enterprise_fields_connectors(); fields("node_status") -> [ diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index bb3ca87b7..6e520ba58 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -412,7 +412,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"webhook_name">> => #{ - <<"connector">> => <<"connector_webhook_name">>, + <<"connector">> => <<"webhook_name">>, <<"description">> => <<>>, <<"enable">> => true, <<"parameters">> => @@ -440,7 +440,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"http">> => #{ - <<"connector_webhook_name">> => + <<"webhook_name">> => #{ <<"connect_timeout">> => <<"15s">>, <<"description">> => <<>>, diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index cd8d597de..30e60df2b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -241,6 +241,12 @@ parse_user_properties(<<"${pub_props.'User-Property'}">>) -> %% we do not want to force users to select the value %% the value will be taken from Env.pub_props directly ?ORIGINAL_USER_PROPERTIES; +parse_user_properties(<<"${.pub_props.'User-Property'}">>) -> + %% keep the original + %% avoid processing this special variable because + %% we do not want to force users to select the value + %% the value will be taken from Env.pub_props directly + ?ORIGINAL_USER_PROPERTIES; parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable emqx_template:parse(V); diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 70a7fc32c..b58d877e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -23,6 +23,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([start_link/0]). @@ -482,8 +483,7 @@ with_parsed_rule(Params = #{id := RuleId, sql := Sql, actions := Actions}, Creat ok -> ok; {error, NonExistentBridgeIDs} -> - ?SLOG(error, #{ - msg => "action_references_nonexistent_bridges", + ?tp(error, "action_references_nonexistent_bridges", #{ rule_id => RuleId, nonexistent_bridge_ids => NonExistentBridgeIDs, hint => "this rule will be disabled" @@ -626,7 +626,7 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeID, #{atom_name => false}), case emqx_action_info:is_action_type(Type) of - true -> {action, Type, Name}; + true -> {source, Type, Name}; false -> {bridge_v1, Type, Name} end end, @@ -646,7 +646,8 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul fun({Kind, Type, Name}) -> LookupFn = case Kind of - action -> fun emqx_bridge_v2:lookup/2; + action -> fun emqx_bridge_v2:lookup_action/2; + source -> fun emqx_bridge_v2:lookup_source/2; bridge_v1 -> fun emqx_bridge:lookup/2 end, try diff --git a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon index 7c7bf68c9..25bb8aad2 100644 --- a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon @@ -194,4 +194,9 @@ username.desc: username.label: """Username""" +config_connector.desc: +"""Configurations for an MQTT connector.""" +config_connector.label: +"""MQTT connector""" + } diff --git a/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon new file mode 100644 index 000000000..637189781 --- /dev/null +++ b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon @@ -0,0 +1,22 @@ +emqx_bridge_mqtt_pubsub_schema { + action_parameters.desc: + """Action specific configs.""" + action_parameters.label: + """Action""" + + ingress_parameters.desc: + """Source specific configs.""" + ingress_parameters.label: + """Source""" + + mqtt_publisher_action.desc: + """Action configs.""" + mqtt_publisher_action.label: + """Action""" + + mqtt_subscriber_source.desc: + """Source configs.""" + mqtt_subscriber_source.label: + """Source""" + +} diff --git a/rel/i18n/emqx_bridge_v2_schema.hocon b/rel/i18n/emqx_bridge_v2_schema.hocon index 69f8a9109..3a4bf6140 100644 --- a/rel/i18n/emqx_bridge_v2_schema.hocon +++ b/rel/i18n/emqx_bridge_v2_schema.hocon @@ -1,10 +1,16 @@ emqx_bridge_v2_schema { desc_bridges_v2.desc: -"""Configuration for bridges.""" +"""Configuration for actions.""" desc_bridges_v2.label: -"""Bridge Configuration""" +"""Action Configuration""" + +desc_sources.desc: +"""Configuration for sources.""" + +desc_sources.label: +"""Source Configuration""" mqtt_topic.desc: """MQTT topic or topic filter as data source (action input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in the remote system."""