From af3a6043541e2721d6f078bb526fd31bab11888f Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 18 Mar 2024 13:36:27 +0100 Subject: [PATCH 01/23] refactor: add emqx_connector_info behavior This commit adds the behavior `emqx_connector_info`. The `emqx_connector_info` behavior should be implement when creating a new connector to provide information about the connector (such as connector schema etc) to the `emqx_connetor` application. The connector in the `emqx_bridge_dynamo` application has also been refactored to use the new behavior (as a test to see that the behavior is working as intended). Fixes: https://emqx.atlassian.net/browse/EMQX-11427 --- .../src/emqx_bridge_dynamo.app.src | 5 +- .../src/emqx_bridge_dynamo_connector_info.erl | 43 ++++ .../src/emqx_connector_info.erl | 193 ++++++++++++++++++ .../src/schema/emqx_connector_ee_schema.erl | 21 +- .../src/schema/emqx_connector_schema.erl | 39 +++- 5 files changed, 277 insertions(+), 24 deletions(-) create mode 100644 apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_info.erl create mode 100644 apps/emqx_connector/src/emqx_connector_info.erl diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index 580f4eebc..81e7fc128 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -8,7 +8,10 @@ emqx_resource, erlcloud ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_dynamo_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_dynamo_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_dynamo_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_info.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_info.erl new file mode 100644 index 000000000..f82420338 --- /dev/null +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_dynamo_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + dynamo. + +bridge_types() -> + [dynamo]. + +resource_callback_module() -> + emqx_bridge_dynamo_connector. + +config_schema() -> + {dynamo, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_dynamo, "config_connector")), + #{ + desc => <<"DynamoDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_dynamo. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_dynamo, <<"dynamo">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl new file mode 100644 index 000000000..d3e7dd27e --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -0,0 +1,193 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The module which knows everything about connectors. + +-module(emqx_connector_info). + +%% Public API +-export([ + connector_types/0, + bridge_types/1, + resource_callback_module/1, + schema_module/1, + config_schema/1, + api_schema/2 +]). + +-export([clean_cache/0]). + +-callback type_name() -> atom(). +-callback bridge_types() -> [atom()]. +-callback resource_callback_module() -> atom(). +-callback schema_module() -> atom(). +-callback config_schema() -> term(). +-callback api_schema([char()]) -> term(). + +%% ==================================================================== +%% HardCoded list of info modules for connectors +%% TODO: Remove this list once we have made sure that all relevants +%% apps are loaded before this module is called. +%% ==================================================================== + +-if(?EMQX_RELEASE_EDITION == ee). +hard_coded_connector_info_modules_ee() -> + [ + emqx_bridge_dynamo_connector_info + ]. +-else. +hard_coded_connector_info_modules_ee() -> + []. +-endif. + +hard_coded_connector_info_modules_common() -> + []. + +hard_coded_connector_info_modules() -> + hard_coded_connector_info_modules_common() ++ hard_coded_connector_info_modules_ee(). + +%% -------------------------------------------------------------------- +%% Atom macros to avoid typos +%% -------------------------------------------------------------------- + +-define(emqx_connector_info_modules, emqx_connector_info_modules). +-define(connector_type_to_info_module, connector_type_to_info_module). +-define(connector_type_names, connector_type_names). +-define(connector_type_to_bridge_types, connector_type_to_bridge_types). +-define(connector_type_to_resource_callback_module, connector_type_to_resource_callback_module). +-define(connector_type_to_schema_module, connector_type_to_schema_module). +-define(connector_type_to_config_schema, connector_type_to_config_schema). + +%% ==================================================================== +%% API +%% ==================================================================== + +connector_types() -> + InfoMap = info_map(), + TypeNamesMap = maps:get(?connector_type_names, InfoMap), + maps:keys(TypeNamesMap). + +bridge_types(ConnectorType) -> + InfoMap = info_map(), + ConToBridges = maps:get(?connector_type_to_bridge_types, InfoMap), + maps:get(ConnectorType, ConToBridges). + +resource_callback_module(ConnectorType) -> + InfoMap = info_map(), + ConToCallbackMod = maps:get(?connector_type_to_resource_callback_module, InfoMap), + maps:get(ConnectorType, ConToCallbackMod). + +schema_module(ConnectorType) -> + InfoMap = info_map(), + ConToSchemaMod = maps:get(?connector_type_to_schema_module, InfoMap), + maps:get(ConnectorType, ConToSchemaMod). + +config_schema(ConnectorType) -> + InfoMap = info_map(), + ConToConfSchema = maps:get(?connector_type_to_config_schema, InfoMap), + maps:get(ConnectorType, ConToConfSchema). + +api_schema(ConnectorType, Method) -> + InfoMod = get_info_module(ConnectorType), + InfoMod:api_schema(Method). + +%% ==================================================================== +%% Internal functions for building the info map and accessing it +%% ==================================================================== + +get_info_module(ConnectorType) -> + InfoMap = info_map(), + ConToInfoMod = maps:get(?connector_type_to_info_module, InfoMap), + maps:get(ConnectorType, ConToInfoMod). + +internal_emqx_connector_persistent_term_info_key() -> + ?FUNCTION_NAME. + +info_map() -> + case persistent_term:get(internal_emqx_connector_persistent_term_info_key(), not_found) of + not_found -> + build_cache(); + InfoMap -> + InfoMap + end. + +build_cache() -> + InfoModules = connector_info_modules(), + InfoMap = + lists:foldl( + fun(Module, InfoMapSoFar) -> + ModuleInfoMap = get_info_map(Module), + emqx_utils_maps:deep_merge(InfoMapSoFar, ModuleInfoMap) + end, + initial_info_map(), + InfoModules + ), + %% Update the persistent term with the new info map + persistent_term:put(internal_emqx_connector_persistent_term_info_key(), InfoMap), + InfoMap. + +clean_cache() -> + persistent_term:erase(internal_emqx_connector_persistent_term_info_key()). + +connector_info_modules() -> + InfoModules = [ + connector_info_modules(App) + || {App, _, _} <- application:loaded_applications() + ], + lists:usort(lists:flatten(InfoModules) ++ hard_coded_connector_info_modules()). + +connector_info_modules(App) -> + case application:get_env(App, ?emqx_connector_info_modules) of + {ok, Modules} -> + Modules; + _ -> + [] + end. + +initial_info_map() -> + #{ + ?connector_type_names => #{}, + ?connector_type_to_info_module => #{}, + ?connector_type_to_bridge_types => #{}, + ?connector_type_to_resource_callback_module => #{}, + ?connector_type_to_schema_module => #{}, + ?connector_type_to_config_schema => #{} + }. + +get_info_map(Module) -> + %% Force the module to get loaded + _ = code:ensure_loaded(Module), + Type = Module:type_name(), + #{ + ?connector_type_names => #{ + Type => true + }, + ?connector_type_to_info_module => #{ + Type => Module + }, + ?connector_type_to_bridge_types => #{ + Type => Module:bridge_types() + }, + ?connector_type_to_resource_callback_module => #{ + Type => Module:resource_callback_module() + }, + ?connector_type_to_schema_module => #{ + Type => Module:schema_module() + }, + ?connector_type_to_config_schema => #{ + Type => Module:config_schema() + } + }. diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index d45d5de35..bf5d9d619 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -26,8 +26,6 @@ resource_type(azure_event_hub_producer) -> emqx_bridge_kafka_impl_producer; resource_type(confluent_producer) -> emqx_bridge_kafka_impl_producer; -resource_type(dynamo) -> - emqx_bridge_dynamo_connector; resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer; resource_type(gcp_pubsub_producer) -> @@ -85,7 +83,12 @@ resource_type(rabbitmq) -> resource_type(s3) -> emqx_bridge_s3_connector; resource_type(Type) -> - error({unknown_connector_type, Type}). + try + emqx_connector_info:resource_callback_module(Type) + catch + _:_ -> + error({unknown_connector_type, Type}) + end. %% For connectors that need to override connector configurations. connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> @@ -132,14 +135,6 @@ connector_structs() -> required => false } )}, - {dynamo, - mk( - hoconsc:map(name, ref(emqx_bridge_dynamo, "config_connector")), - #{ - desc => <<"DynamoDB Connector Config">>, - required => false - } - )}, {gcp_pubsub_consumer, mk( hoconsc:map(name, ref(emqx_bridge_gcp_pubsub_consumer_schema, "config_connector")), @@ -371,7 +366,6 @@ schema_modules() -> [ emqx_bridge_azure_event_hub, emqx_bridge_confluent_producer, - emqx_bridge_dynamo, emqx_bridge_gcp_pubsub_consumer_schema, emqx_bridge_gcp_pubsub_producer_schema, emqx_bridge_hstreamdb, @@ -412,9 +406,6 @@ api_schemas(Method) -> api_ref( emqx_bridge_confluent_producer, <<"confluent_producer">>, Method ++ "_connector" ), - api_ref( - emqx_bridge_dynamo, <<"dynamo">>, Method ++ "_connector" - ), api_ref( emqx_bridge_gcp_pubsub_consumer_schema, <<"gcp_pubsub_consumer">>, diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index d68514c41..4ce386534 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -29,7 +29,8 @@ transform_bridges_v1_to_connectors_and_bridges_v2/1, transform_bridge_v1_config_to_action_config/4, top_level_common_connector_keys/0, - project_to_connector_resource_opts/1 + project_to_connector_resource_opts/1, + api_ref/3 ]). -export([roots/0, fields/1, desc/1, namespace/0, tags/0]). @@ -112,12 +113,19 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ - emqx_connector_ee_schema:schema_modules(). + emqx_connector_ee_schema:schema_modules() ++ connector_info_schema_modules(). -else. schema_modules() -> - [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema]. + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ connector_info_schema_modules(). -endif. +connector_info_schema_modules() -> + ConnectorTypes = emqx_connector_info:connector_types(), + [ + emqx_connector_info:schema_module(Type) + || Type <- ConnectorTypes + ]. + %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. connector_type_to_bridge_types(http) -> @@ -126,8 +134,6 @@ connector_type_to_bridge_types(azure_event_hub_producer) -> [azure_event_hub_producer]; connector_type_to_bridge_types(confluent_producer) -> [confluent_producer]; -connector_type_to_bridge_types(dynamo) -> - [dynamo]; connector_type_to_bridge_types(gcp_pubsub_consumer) -> [gcp_pubsub_consumer]; connector_type_to_bridge_types(gcp_pubsub_producer) -> @@ -185,7 +191,9 @@ connector_type_to_bridge_types(tdengine) -> connector_type_to_bridge_types(rabbitmq) -> [rabbitmq]; connector_type_to_bridge_types(s3) -> - [s3]. + [s3]; +connector_type_to_bridge_types(Type) -> + emqx_connector_info:bridge_types(Type). actions_config_name(action) -> <<"actions">>; actions_config_name(source) -> <<"sources">>. @@ -481,7 +489,15 @@ post_request() -> api_schema(Method) -> CE = api_schemas(Method), EE = enterprise_api_schemas(Method), - hoconsc:union(connector_api_union(CE ++ EE)). + InfoModSchemas = emqx_connector_info_api_schemas(Method), + hoconsc:union(connector_api_union(CE ++ EE ++ InfoModSchemas)). + +emqx_connector_info_api_schemas(Method) -> + ConnectorTypes = emqx_connector_info:connector_types(), + [ + emqx_connector_info:api_schema(Type, Method) + || Type <- ConnectorTypes + ]. connector_api_union(Refs) -> Index = maps:from_list(Refs), @@ -544,7 +560,7 @@ fields(connectors) -> required => false } )} - ] ++ enterprise_fields_connectors(); + ] ++ enterprise_fields_connectors() ++ connector_info_fields_connectors(); fields("node_status") -> [ node_name(), @@ -557,6 +573,13 @@ fields("node_status") -> })} ]. +connector_info_fields_connectors() -> + ConnectorTypes = emqx_connector_info:connector_types(), + [ + emqx_connector_info:config_schema(Type) + || Type <- ConnectorTypes + ]. + desc(connectors) -> ?DESC("desc_connectors"); desc("node_status") -> From 795b668c4f048dc1902ef08cd5bb660dc58443e2 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 18 Mar 2024 16:51:32 +0100 Subject: [PATCH 02/23] refactor(azure_event_hub connector): to use emqx_connector_info This commit refactors the emqx_bridge_azure_event_hub to use the emqx_connector_info behavior. The emqx_bridge_azure_event_hub related information can thus be removed from emqx_connector_ee_schema and emqx_connector_schema. --- .../src/emqx_bridge_azure_event_hub.app.src | 7 ++- ..._bridge_azure_event_hub_connector_info.erl | 58 +++++++++++++++++++ .../src/emqx_connector_info.erl | 38 ++++++++++-- .../src/emqx_connector_resource.erl | 15 ++++- .../src/schema/emqx_connector_ee_schema.erl | 17 ------ .../src/schema/emqx_connector_schema.erl | 2 - 6 files changed, 108 insertions(+), 29 deletions(-) create mode 100644 apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_connector_info.erl diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src index b588d1e18..7e70fffff 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_azure_event_hub, [ {description, "EMQX Enterprise Azure Event Hub Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, @@ -9,7 +9,10 @@ telemetry, wolff ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_azure_event_hub_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_azure_event_hub_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_azure_event_hub_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_connector_info.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_connector_info.erl new file mode 100644 index 000000000..932df54fa --- /dev/null +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_connector_info.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @doc +%% Implementation of emqx_connector_info for Azure Event Hub connector. +%% This module provides connector-specific information and configurations +%% required by the emqx_connector application. +%%-------------------------------------------------------------------- +-module(emqx_bridge_azure_event_hub_connector_info). + +-behaviour(emqx_connector_info). + +%% API exports. +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +%%-------------------------------------------------------------------- +%% API Functions +%%-------------------------------------------------------------------- + +type_name() -> + azure_event_hub_producer. + +bridge_types() -> + [azure_event_hub_producer]. + +resource_callback_module() -> + emqx_bridge_kafka_impl_producer. + +config_transform_module() -> + emqx_bridge_azure_event_hub. + +config_schema() -> + {azure_event_hub_producer, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_azure_event_hub, "config_connector")), + #{ + desc => <<"Azure Event Hub Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_azure_event_hub. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_azure_event_hub, <<"azure_event_hub_producer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index d3e7dd27e..2589726d4 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -25,7 +25,8 @@ resource_callback_module/1, schema_module/1, config_schema/1, - api_schema/2 + api_schema/2, + config_transform_module/1 ]). -export([clean_cache/0]). @@ -36,6 +37,15 @@ -callback schema_module() -> atom(). -callback config_schema() -> term(). -callback api_schema([char()]) -> term(). +%% Optional callback that should return a module with an exported +%% connector_config/2 function. If present this function will be used to +%% transfrom the connector configuration. See the callback connector_config/2 +%% in emqx_connector_resource for more information. +-callback config_transform_module() -> atom(). + +-optional_callbacks([ + config_transform_module/0 +]). %% ==================================================================== %% HardCoded list of info modules for connectors @@ -46,7 +56,8 @@ -if(?EMQX_RELEASE_EDITION == ee). hard_coded_connector_info_modules_ee() -> [ - emqx_bridge_dynamo_connector_info + emqx_bridge_dynamo_connector_info, + emqx_bridge_azure_event_hub_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> @@ -70,6 +81,7 @@ hard_coded_connector_info_modules() -> -define(connector_type_to_resource_callback_module, connector_type_to_resource_callback_module). -define(connector_type_to_schema_module, connector_type_to_schema_module). -define(connector_type_to_config_schema, connector_type_to_config_schema). +-define(connector_type_to_config_transform_module, connector_type_to_config_transform_module). %% ==================================================================== %% API @@ -101,14 +113,19 @@ config_schema(ConnectorType) -> maps:get(ConnectorType, ConToConfSchema). api_schema(ConnectorType, Method) -> - InfoMod = get_info_module(ConnectorType), + InfoMod = info_module(ConnectorType), InfoMod:api_schema(Method). +config_transform_module(ConnectorType) -> + InfoMap = info_map(), + ConToConfTransMod = maps:get(?connector_type_to_config_transform_module, InfoMap), + maps:get(ConnectorType, ConToConfTransMod, undefined). + %% ==================================================================== %% Internal functions for building the info map and accessing it %% ==================================================================== -get_info_module(ConnectorType) -> +info_module(ConnectorType) -> InfoMap = info_map(), ConToInfoMod = maps:get(?connector_type_to_info_module, InfoMap), maps:get(ConnectorType, ConToInfoMod). @@ -164,13 +181,21 @@ initial_info_map() -> ?connector_type_to_bridge_types => #{}, ?connector_type_to_resource_callback_module => #{}, ?connector_type_to_schema_module => #{}, - ?connector_type_to_config_schema => #{} + ?connector_type_to_config_schema => #{}, + ?connector_type_to_config_transform_module => #{} }. get_info_map(Module) -> %% Force the module to get loaded _ = code:ensure_loaded(Module), Type = Module:type_name(), + ConfigTransformModule = + case erlang:function_exported(Module, config_transform_module, 0) of + true -> + Module:config_transform_module(); + false -> + undefined + end, #{ ?connector_type_names => #{ Type => true @@ -189,5 +214,8 @@ get_info_map(Module) -> }, ?connector_type_to_config_schema => #{ Type => Module:config_schema() + }, + ?connector_type_to_config_transform_module => #{ + Type => ConfigTransformModule } }. diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index fbebc20f5..1b434496a 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -67,15 +67,24 @@ connector_to_resource_type(ConnectorType) -> connector_to_resource_type_ce(ConnectorType) end. +connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> + connector_impl_module(binary_to_atom(ConnectorType, utf8)); connector_impl_module(ConnectorType) -> - emqx_connector_ee_schema:connector_impl_module(ConnectorType). + case emqx_connector_ee_schema:connector_impl_module(ConnectorType) of + undefined -> + emqx_connector_info:config_transform_module(ConnectorType); + Module -> + Module + end. -else. connector_to_resource_type(ConnectorType) -> connector_to_resource_type_ce(ConnectorType). -connector_impl_module(_ConnectorType) -> - undefined. +connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> + connector_impl_module(binary_to_atom(ConnectorType, utf8)); +connector_impl_module(ConnectorType) -> + emqx_connector_info:config_transform_module(ConnectorType). -endif. diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index bf5d9d619..f7fd3db7a 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,9 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(azure_event_hub_producer) -> - %% We use AEH's Kafka interface. - emqx_bridge_kafka_impl_producer; resource_type(confluent_producer) -> emqx_bridge_kafka_impl_producer; resource_type(gcp_pubsub_consumer) -> @@ -93,8 +90,6 @@ resource_type(Type) -> %% For connectors that need to override connector configurations. connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(azure_event_hub_producer) -> - emqx_bridge_azure_event_hub; connector_impl_module(confluent_producer) -> emqx_bridge_confluent_producer; connector_impl_module(iotdb) -> @@ -119,14 +114,6 @@ fields(connectors) -> connector_structs() -> [ - {azure_event_hub_producer, - mk( - hoconsc:map(name, ref(emqx_bridge_azure_event_hub, "config_connector")), - #{ - desc => <<"Azure Event Hub Connector Config">>, - required => false - } - )}, {confluent_producer, mk( hoconsc:map(name, ref(emqx_bridge_confluent_producer, "config_connector")), @@ -364,7 +351,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_azure_event_hub, emqx_bridge_confluent_producer, emqx_bridge_gcp_pubsub_consumer_schema, emqx_bridge_gcp_pubsub_producer_schema, @@ -400,9 +386,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref( - emqx_bridge_azure_event_hub, <<"azure_event_hub_producer">>, Method ++ "_connector" - ), api_ref( emqx_bridge_confluent_producer, <<"confluent_producer">>, Method ++ "_connector" ), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 4ce386534..b68dc194c 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -130,8 +130,6 @@ connector_info_schema_modules() -> %% from the latest connector type name. connector_type_to_bridge_types(http) -> [webhook, http]; -connector_type_to_bridge_types(azure_event_hub_producer) -> - [azure_event_hub_producer]; connector_type_to_bridge_types(confluent_producer) -> [confluent_producer]; connector_type_to_bridge_types(gcp_pubsub_consumer) -> From 76d32b6e970fe4ffb9acd98b11678e77146cbe84 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 18 Mar 2024 17:16:18 +0100 Subject: [PATCH 03/23] refactor(confluent connector): to use emqx_connector_info This commit refactors the emqx_bridge_confluent to use the emqx_connector_info behavior. The emqx_bridge_confluent related information can thus be removed from emqx_connector_ee_schema and emqx_connector_schema. --- .../src/emqx_bridge_confluent.app.src | 7 ++- ...idge_confluent_producer_connector_info.erl | 52 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 16 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer_connector_info.erl diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src index e63b8df6a..46d6617c3 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_confluent, [ {description, "EMQX Enterprise Confluent Connector and Action"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, @@ -9,7 +9,10 @@ telemetry, wolff ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_confluent_producer_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_confluent_producer_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_confluent_producer_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer_connector_info.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer_connector_info.erl new file mode 100644 index 000000000..6c9aa5d90 --- /dev/null +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer_connector_info.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_confluent_producer_connector_info). + +-behaviour(emqx_connector_info). + +%% API exports. +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +%%-------------------------------------------------------------------- +%% API Functions +%%-------------------------------------------------------------------- + +type_name() -> + confluent_producer. + +bridge_types() -> + [confluent_producer]. + +resource_callback_module() -> + emqx_bridge_kafka_impl_producer. + +config_transform_module() -> + emqx_bridge_confluent_producer. + +config_schema() -> + {confluent_producer, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_confluent_producer, "config_connector")), + #{ + desc => <<"Confluent Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_confluent_producer. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_confluent_producer, <<"confluent_producer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 2589726d4..954cbc882 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -57,7 +57,8 @@ hard_coded_connector_info_modules_ee() -> [ emqx_bridge_dynamo_connector_info, - emqx_bridge_azure_event_hub_connector_info + emqx_bridge_azure_event_hub_connector_info, + emqx_bridge_confluent_producer_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index f7fd3db7a..54cc4b1be 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(confluent_producer) -> - emqx_bridge_kafka_impl_producer; resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer; resource_type(gcp_pubsub_producer) -> @@ -90,8 +88,6 @@ resource_type(Type) -> %% For connectors that need to override connector configurations. connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(confluent_producer) -> - emqx_bridge_confluent_producer; connector_impl_module(iotdb) -> emqx_bridge_iotdb_connector; connector_impl_module(elasticsearch) -> @@ -114,14 +110,6 @@ fields(connectors) -> connector_structs() -> [ - {confluent_producer, - mk( - hoconsc:map(name, ref(emqx_bridge_confluent_producer, "config_connector")), - #{ - desc => <<"Confluent Connector Config">>, - required => false - } - )}, {gcp_pubsub_consumer, mk( hoconsc:map(name, ref(emqx_bridge_gcp_pubsub_consumer_schema, "config_connector")), @@ -351,7 +339,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_confluent_producer, emqx_bridge_gcp_pubsub_consumer_schema, emqx_bridge_gcp_pubsub_producer_schema, emqx_bridge_hstreamdb, @@ -386,9 +373,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref( - emqx_bridge_confluent_producer, <<"confluent_producer">>, Method ++ "_connector" - ), api_ref( emqx_bridge_gcp_pubsub_consumer_schema, <<"gcp_pubsub_consumer">>, diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index b68dc194c..8adf634eb 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -130,8 +130,6 @@ connector_info_schema_modules() -> %% from the latest connector type name. connector_type_to_bridge_types(http) -> [webhook, http]; -connector_type_to_bridge_types(confluent_producer) -> - [confluent_producer]; connector_type_to_bridge_types(gcp_pubsub_consumer) -> [gcp_pubsub_consumer]; connector_type_to_bridge_types(gcp_pubsub_producer) -> From d0bebe8b93be0813443f4bff5df8dbe747e11df8 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 18 Mar 2024 17:43:50 +0100 Subject: [PATCH 04/23] refactor(gcp_pubsub connectors): to use emqx_connector_info This commit refactors the emqx_bridge_gcp_pubsub app to use the emqx_connector_info behavior. The emqx_bridge_gcp_pubsub related information can thus be removed from emqx_connector_ee_schema and emqx_connector_schema. --- .../src/emqx_bridge_gcp_pubsub.app.src | 4 ++ ...dge_gcp_pubsub_consumer_connector_info.erl | 45 +++++++++++++++++++ ...dge_gcp_pubsub_producer_connector_info.erl | 45 +++++++++++++++++++ .../src/emqx_connector_info.erl | 4 +- .../src/schema/emqx_connector_ee_schema.erl | 32 ------------- .../src/schema/emqx_connector_schema.erl | 4 -- 6 files changed, 97 insertions(+), 37 deletions(-) create mode 100644 apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_connector_info.erl create mode 100644 apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_connector_info.erl diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index bd98c43d6..cfa0f95d1 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -12,6 +12,10 @@ {emqx_action_info_modules, [ emqx_bridge_gcp_pubsub_producer_action_info, emqx_bridge_gcp_pubsub_consumer_action_info + ]}, + {emqx_connector_info_modules, [ + emqx_bridge_gcp_pubsub_producer_connector_info, + emqx_bridge_gcp_pubsub_consumer_connector_info ]} ]}, {modules, []}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_connector_info.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_connector_info.erl new file mode 100644 index 000000000..0b9406a1b --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_connector_info.erl @@ -0,0 +1,45 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_gcp_pubsub_consumer_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + gcp_pubsub_consumer. + +bridge_types() -> + [gcp_pubsub_consumer]. + +resource_callback_module() -> + emqx_bridge_gcp_pubsub_impl_consumer. + +config_schema() -> + {gcp_pubsub_consumer, + hoconsc:mk( + hoconsc:map( + name, hoconsc:ref(emqx_bridge_gcp_pubsub_consumer_schema, "config_connector") + ), + #{ + desc => <<"GCP PubSub Consumer Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_gcp_pubsub_consumer_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_gcp_pubsub_consumer_schema, <<"gcp_pubsub_consumer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_connector_info.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_connector_info.erl new file mode 100644 index 000000000..1e6c4ee46 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_connector_info.erl @@ -0,0 +1,45 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_gcp_pubsub_producer_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + gcp_pubsub_producer. + +bridge_types() -> + [gcp_pubsub, gcp_pubsub_producer]. + +resource_callback_module() -> + emqx_bridge_gcp_pubsub_impl_producer. + +config_schema() -> + {gcp_pubsub_producer, + hoconsc:mk( + hoconsc:map( + name, hoconsc:ref(emqx_bridge_gcp_pubsub_producer_schema, "config_connector") + ), + #{ + desc => <<"GCP PubSub Producer Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_gcp_pubsub_producer_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_gcp_pubsub_producer_schema, <<"gcp_pubsub_producer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 954cbc882..86def6da7 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -58,7 +58,9 @@ hard_coded_connector_info_modules_ee() -> [ emqx_bridge_dynamo_connector_info, emqx_bridge_azure_event_hub_connector_info, - emqx_bridge_confluent_producer_connector_info + emqx_bridge_confluent_producer_connector_info, + emqx_bridge_gcp_pubsub_consumer_connector_info, + emqx_bridge_gcp_pubsub_producer_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 54cc4b1be..1ae8794cf 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,10 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(gcp_pubsub_consumer) -> - emqx_bridge_gcp_pubsub_impl_consumer; -resource_type(gcp_pubsub_producer) -> - emqx_bridge_gcp_pubsub_impl_producer; resource_type(hstreamdb) -> emqx_bridge_hstreamdb_connector; resource_type(kafka_consumer) -> @@ -110,22 +106,6 @@ fields(connectors) -> connector_structs() -> [ - {gcp_pubsub_consumer, - mk( - hoconsc:map(name, ref(emqx_bridge_gcp_pubsub_consumer_schema, "config_connector")), - #{ - desc => <<"GCP PubSub Consumer Connector Config">>, - required => false - } - )}, - {gcp_pubsub_producer, - mk( - hoconsc:map(name, ref(emqx_bridge_gcp_pubsub_producer_schema, "config_connector")), - #{ - desc => <<"GCP PubSub Producer Connector Config">>, - required => false - } - )}, {hstreamdb, mk( hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config_connector")), @@ -339,8 +319,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_gcp_pubsub_consumer_schema, - emqx_bridge_gcp_pubsub_producer_schema, emqx_bridge_hstreamdb, emqx_bridge_kafka, emqx_bridge_kafka_consumer_schema, @@ -373,16 +351,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref( - emqx_bridge_gcp_pubsub_consumer_schema, - <<"gcp_pubsub_consumer">>, - Method ++ "_connector" - ), - api_ref( - emqx_bridge_gcp_pubsub_producer_schema, - <<"gcp_pubsub_producer">>, - Method ++ "_connector" - ), api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method ++ "_connector"), api_ref(emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector"), api_ref(emqx_bridge_kafka_consumer_schema, <<"kafka_consumer">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 8adf634eb..4b8cf01e6 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -130,10 +130,6 @@ connector_info_schema_modules() -> %% from the latest connector type name. connector_type_to_bridge_types(http) -> [webhook, http]; -connector_type_to_bridge_types(gcp_pubsub_consumer) -> - [gcp_pubsub_consumer]; -connector_type_to_bridge_types(gcp_pubsub_producer) -> - [gcp_pubsub, gcp_pubsub_producer]; connector_type_to_bridge_types(hstreamdb) -> [hstreamdb]; connector_type_to_bridge_types(kafka_consumer) -> From dc08fab0e82d7c0b0a6412189e2bb6ab9d9cfd96 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 19 Mar 2024 10:11:41 +0100 Subject: [PATCH 05/23] refactor(hstreamdb connector): to use emqx_connector_info This commit refactors the `emqx_bridge_hstreamdb` to use the `emqx_connector_info` behavior. The `emqx_bridge_hstreamdb` related information can thus be removed from `emqx_connector_ee_schema` and `emqx_connector_schema`. --- .../src/emqx_bridge_hstreamdb.app.src | 5 ++- .../emqx_bridge_hstreamdb_connector_info.erl | 43 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector_info.erl diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 84c09fe3a..f9f80b1d0 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -8,7 +8,10 @@ emqx_resource, hstreamdb_erl ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_hstreamdb_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_hstreamdb_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_hstreamdb_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector_info.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector_info.erl new file mode 100644 index 000000000..8fc4f61bf --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_hstreamdb_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + hstreamdb. + +bridge_types() -> + [hstreamdb]. + +resource_callback_module() -> + emqx_bridge_hstreamdb_connector. + +config_schema() -> + {hstreamdb, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_hstreamdb, "config_connector")), + #{ + desc => <<"HStreamDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_hstreamdb. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_hstreamdb, <<"hstreamdb">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 86def6da7..30287a0a1 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -60,7 +60,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_azure_event_hub_connector_info, emqx_bridge_confluent_producer_connector_info, emqx_bridge_gcp_pubsub_consumer_connector_info, - emqx_bridge_gcp_pubsub_producer_connector_info + emqx_bridge_gcp_pubsub_producer_connector_info, + emqx_bridge_hstreamdb_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 1ae8794cf..0e6634d79 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(hstreamdb) -> - emqx_bridge_hstreamdb_connector; resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; resource_type(kafka_producer) -> @@ -106,14 +104,6 @@ fields(connectors) -> connector_structs() -> [ - {hstreamdb, - mk( - hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config_connector")), - #{ - desc => <<"HStreamDB Connector Config">>, - required => false - } - )}, {kafka_consumer, mk( hoconsc:map(name, ref(emqx_bridge_kafka_consumer_schema, "config_connector")), @@ -319,7 +309,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_hstreamdb, emqx_bridge_kafka, emqx_bridge_kafka_consumer_schema, emqx_bridge_kinesis, @@ -351,7 +340,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method ++ "_connector"), api_ref(emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector"), api_ref(emqx_bridge_kafka_consumer_schema, <<"kafka_consumer">>, Method ++ "_connector"), api_ref(emqx_bridge_kinesis, <<"kinesis">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 4b8cf01e6..ee52c7361 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -130,8 +130,6 @@ connector_info_schema_modules() -> %% from the latest connector type name. connector_type_to_bridge_types(http) -> [webhook, http]; -connector_type_to_bridge_types(hstreamdb) -> - [hstreamdb]; connector_type_to_bridge_types(kafka_consumer) -> [kafka_consumer]; connector_type_to_bridge_types(kafka_producer) -> From b8ef357fef8c1ee912941879800b81a721720b00 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 19 Mar 2024 11:28:42 +0100 Subject: [PATCH 06/23] refactor(http connector): to use emqx_connector_info This commit refactors the `emqx_bridge_http` to use the `emqx_connector_info` behavior. The `emqx_bridge_http` related information can thus be removed from `emqx_connector_chema` and `emqx_connector_resource`. --- .../src/emqx_bridge_http.app.src | 5 +- .../src/emqx_bridge_http_connector_info.erl | 56 +++++++++++++++++++ .../src/emqx_connector_info.erl | 2 +- .../src/emqx_connector_resource.erl | 9 ++- .../src/schema/emqx_connector_ee_schema.erl | 7 +-- .../src/schema/emqx_connector_schema.erl | 16 +----- 6 files changed, 70 insertions(+), 25 deletions(-) create mode 100644 apps/emqx_bridge_http/src/emqx_bridge_http_connector_info.erl diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index 681095a08..52899d2a0 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -3,7 +3,10 @@ {vsn, "0.2.4"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, ehttpc]}, - {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_http_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_http_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector_info.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector_info.erl new file mode 100644 index 000000000..f424e2fe4 --- /dev/null +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector_info.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_http_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + http. + +bridge_types() -> + [webhook, http]. + +resource_callback_module() -> + emqx_bridge_http_connector. + +config_schema() -> + {http, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_http_schema, "config_connector")), + #{ + alias => [webhook], + desc => <<"HTTP Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_http_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_http_schema, <<"http">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 30287a0a1..9ab09e821 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -69,7 +69,7 @@ hard_coded_connector_info_modules_ee() -> -endif. hard_coded_connector_info_modules_common() -> - []. + [emqx_bridge_http_connector_info]. hard_coded_connector_info_modules() -> hard_coded_connector_info_modules_common() ++ hard_coded_connector_info_modules_ee(). diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 1b434496a..dec4ab2d7 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -88,14 +88,17 @@ connector_impl_module(ConnectorType) -> -endif. -connector_to_resource_type_ce(http) -> - emqx_bridge_http_connector; connector_to_resource_type_ce(mqtt) -> emqx_bridge_mqtt_connector; % connector_to_resource_type_ce(mqtt_subscriber) -> % emqx_bridge_mqtt_subscriber_connector; connector_to_resource_type_ce(ConnectorType) -> - error({no_bridge_v2, ConnectorType}). + try + emqx_connector_info:resource_callback_module(ConnectorType) + catch + _:_ -> + error({unknown_connector_type, ConnectorType}) + end. resource_id(ConnectorId) when is_binary(ConnectorId) -> <<"connector:", ConnectorId/binary>>. diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 0e6634d79..d2ccdff61 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -72,12 +72,7 @@ resource_type(rabbitmq) -> resource_type(s3) -> emqx_bridge_s3_connector; resource_type(Type) -> - try - emqx_connector_info:resource_callback_module(Type) - catch - _:_ -> - error({unknown_connector_type, Type}) - end. + error({unknown_connector_type, Type}). %% For connectors that need to override connector configurations. connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index ee52c7361..edac6976d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -91,7 +91,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector"), api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector") ]. @@ -112,11 +111,11 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ + [emqx_bridge_mqtt_connector_schema] ++ emqx_connector_ee_schema:schema_modules() ++ connector_info_schema_modules(). -else. schema_modules() -> - [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ connector_info_schema_modules(). + [emqx_bridge_mqtt_connector_schema] ++ connector_info_schema_modules(). -endif. connector_info_schema_modules() -> @@ -128,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(http) -> - [webhook, http]; connector_type_to_bridge_types(kafka_consumer) -> [kafka_consumer]; connector_type_to_bridge_types(kafka_producer) -> @@ -533,15 +530,6 @@ roots() -> fields(connectors) -> [ - {http, - mk( - hoconsc:map(name, ref(emqx_bridge_http_schema, "config_connector")), - #{ - alias => [webhook], - desc => <<"HTTP Connector Config">>, - required => false - } - )}, {mqtt, mk( hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), From a3e631cda2b6d47d63961a23d77e91b19bd0f046 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 19 Mar 2024 13:46:03 +0100 Subject: [PATCH 07/23] refactor(kafka connectors): to use emqx_connector_info This commit refactors the `emqx_bridge_kafka` to use the `emqx_connector_info` behavior. The `emqx_bridge_kafka` related information can thus be removed from `emqx_connector_chema` and `emqx_connector_resource`. --- .../src/emqx_bridge_kafka.app.src | 4 ++ .../src/emqx_bridge_kafka_connector_info.erl | 43 +++++++++++++++++++ ...x_bridge_kafka_consumer_connector_info.erl | 43 +++++++++++++++++++ .../src/emqx_connector_info.erl | 4 +- .../src/schema/emqx_connector_ee_schema.erl | 24 ----------- .../src/schema/emqx_connector_schema.erl | 4 -- 6 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl create mode 100644 apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_connector_info.erl diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index 74ba58217..90ac23e81 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -16,6 +16,10 @@ {emqx_action_info_modules, [ emqx_bridge_kafka_action_info, emqx_bridge_kafka_consumer_action_info + ]}, + {emqx_connector_info_modules, [ + emqx_bridge_kafka_consumer_connector_info, + emqx_bridge_kafka_connector_info ]} ]}, {modules, []}, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl new file mode 100644 index 000000000..d0b013cc3 --- /dev/null +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_kafka_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + kafka_producer. + +bridge_types() -> + [kafka, kafka_producer]. + +resource_callback_module() -> + emqx_bridge_kafka_impl_producer. + +config_schema() -> + {kafka_producer, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_kafka, "config_connector")), + #{ + desc => <<"Kafka Producer Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_kafka. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_connector_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_connector_info.erl new file mode 100644 index 000000000..fa63dfaaf --- /dev/null +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_kafka_consumer_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + kafka_consumer. + +bridge_types() -> + [kafka_consumer]. + +resource_callback_module() -> + emqx_bridge_kafka_impl_consumer. + +config_schema() -> + {kafka_consumer, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_kafka_consumer_schema, "config_connector")), + #{ + desc => <<"Kafka Consumer Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_kafka_consumer_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_kafka_consumer_schema, <<"kafka_consumer">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 9ab09e821..7b0b6f99b 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -61,7 +61,9 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_confluent_producer_connector_info, emqx_bridge_gcp_pubsub_consumer_connector_info, emqx_bridge_gcp_pubsub_producer_connector_info, - emqx_bridge_hstreamdb_connector_info + emqx_bridge_hstreamdb_connector_info, + emqx_bridge_kafka_consumer_connector_info, + emqx_bridge_kafka_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index d2ccdff61..2323d6788 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,10 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(kafka_consumer) -> - emqx_bridge_kafka_impl_consumer; -resource_type(kafka_producer) -> - emqx_bridge_kafka_impl_producer; resource_type(kinesis) -> emqx_bridge_kinesis_impl_producer; resource_type(matrix) -> @@ -99,22 +95,6 @@ fields(connectors) -> connector_structs() -> [ - {kafka_consumer, - mk( - hoconsc:map(name, ref(emqx_bridge_kafka_consumer_schema, "config_connector")), - #{ - desc => <<"Kafka Consumer Connector Config">>, - required => false - } - )}, - {kafka_producer, - mk( - hoconsc:map(name, ref(emqx_bridge_kafka, "config_connector")), - #{ - desc => <<"Kafka Producer Connector Config">>, - required => false - } - )}, {kinesis, mk( hoconsc:map(name, ref(emqx_bridge_kinesis, "config_connector")), @@ -304,8 +284,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_kafka, - emqx_bridge_kafka_consumer_schema, emqx_bridge_kinesis, emqx_bridge_matrix, emqx_bridge_mongodb, @@ -335,8 +313,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector"), - api_ref(emqx_bridge_kafka_consumer_schema, <<"kafka_consumer">>, Method ++ "_connector"), api_ref(emqx_bridge_kinesis, <<"kinesis">>, Method ++ "_connector"), api_ref(emqx_bridge_matrix, <<"matrix">>, Method ++ "_connector"), api_ref(emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index edac6976d..58308f8e3 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,10 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(kafka_consumer) -> - [kafka_consumer]; -connector_type_to_bridge_types(kafka_producer) -> - [kafka, kafka_producer]; connector_type_to_bridge_types(kinesis) -> [kinesis, kinesis_producer]; connector_type_to_bridge_types(matrix) -> From c3d7d68cfcda3aa643f9b905cbe39c38a4e1be95 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 19 Mar 2024 14:43:27 +0100 Subject: [PATCH 08/23] refactor(kinesis,pgsql,timescale,matrix connectors): emqx_connector_info This commit refactors the kinesis, pgsql, timescale and matrix connectors to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_kinesis.app.src | 1 + .../emqx_bridge_kinesis_connector_info.erl | 43 +++++++++++++++++ .../src/emqx_bridge_matrix.app.src | 7 ++- .../src/emqx_bridge_matrix_connector_info.erl | 43 +++++++++++++++++ .../src/emqx_bridge_pgsql.app.src | 7 ++- .../src/emqx_bridge_pgsql_connector_info.erl | 43 +++++++++++++++++ .../src/emqx_bridge_timescale.app.src | 8 ++-- .../emqx_bridge_timescale_connector_info.erl | 43 +++++++++++++++++ .../src/emqx_connector_info.erl | 6 ++- .../src/schema/emqx_connector_ee_schema.erl | 47 ------------------- .../src/schema/emqx_connector_schema.erl | 8 ---- 11 files changed, 193 insertions(+), 63 deletions(-) create mode 100644 apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_info.erl create mode 100644 apps/emqx_bridge_matrix/src/emqx_bridge_matrix_connector_info.erl create mode 100644 apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql_connector_info.erl create mode 100644 apps/emqx_bridge_timescale/src/emqx_bridge_timescale_connector_info.erl diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index 2e59fa8b2..6f885f27b 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -8,6 +8,7 @@ erlcloud ]}, {env, [{emqx_action_info_modules, [emqx_bridge_kinesis_action_info]}]}, + {env, [{emqx_connector_info_modules, [emqx_bridge_kinesis_connector_info]}]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_info.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_info.erl new file mode 100644 index 000000000..317eeff87 --- /dev/null +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_kinesis_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + kinesis. + +bridge_types() -> + [kinesis, kinesis_producer]. + +resource_callback_module() -> + emqx_bridge_kinesis_impl_producer. + +config_schema() -> + {kinesis, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_kinesis, "config_connector")), + #{ + desc => <<"Kinesis Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_kinesis. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_kinesis, <<"kinesis">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src index 479aa13df..80864ba67 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -1,13 +1,16 @@ {application, emqx_bridge_matrix, [ {description, "EMQX Enterprise MatrixDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, stdlib, emqx_resource ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_matrix_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_matrix_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_matrix_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix_connector_info.erl b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix_connector_info.erl new file mode 100644 index 000000000..cc68b61ff --- /dev/null +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_matrix_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + matrix. + +bridge_types() -> + [matrix]. + +resource_callback_module() -> + emqx_postgresql. + +config_schema() -> + {matrix, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_matrix, "config_connector")), + #{ + desc => <<"Matrix Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_matrix. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_matrix, <<"matrix">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index fafd49f05..d4f97a721 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,10 @@ emqx_resource, emqx_postgresql ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_pgsql_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_pgsql_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_pgsql_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql_connector_info.erl b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql_connector_info.erl new file mode 100644 index 000000000..12cacd185 --- /dev/null +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_pgsql_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + pgsql. + +bridge_types() -> + [pgsql]. + +resource_callback_module() -> + emqx_postgresql. + +config_schema() -> + {pgsql, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_pgsql, "config_connector")), + #{ + desc => <<"PostgreSQL Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_postgresql_connector_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_postgresql_connector_schema, <<"pgsql">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src index 53302a21f..477da7c9e 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_timescale, [ {description, "EMQX Enterprise TimescaleDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource]}, - {env, [{emqx_action_info_module, emqx_bridge_timescale_action_info}]}, - {env, []}, + {env, [ + {emqx_action_info_modules, emqx_bridge_timescale_action_info}, + {emqx_connector_info_modules, emqx_bridge_timescale_connector_info} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale_connector_info.erl b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale_connector_info.erl new file mode 100644 index 000000000..1d607538b --- /dev/null +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_timescale_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + timescale. + +bridge_types() -> + [timescale]. + +resource_callback_module() -> + emqx_postgresql. + +config_schema() -> + {timescale, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_timescale, "config_connector")), + #{ + desc => <<"Timescale Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_timescale. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_timescale, <<"timescale">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 7b0b6f99b..538ecce58 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -63,7 +63,11 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_gcp_pubsub_producer_connector_info, emqx_bridge_hstreamdb_connector_info, emqx_bridge_kafka_consumer_connector_info, - emqx_bridge_kafka_connector_info + emqx_bridge_kafka_connector_info, + emqx_bridge_kinesis_connector_info, + emqx_bridge_matrix_connector_info, + emqx_bridge_pgsql_connector_info, + emqx_bridge_timescale_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 2323d6788..51c510806 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,10 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(kinesis) -> - emqx_bridge_kinesis_impl_producer; -resource_type(matrix) -> - emqx_postgresql; resource_type(mongodb) -> emqx_bridge_mongodb_connector; resource_type(oracle) -> @@ -37,16 +33,12 @@ resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(mysql) -> emqx_bridge_mysql_connector; -resource_type(pgsql) -> - emqx_postgresql; resource_type(syskeeper_forwarder) -> emqx_bridge_syskeeper_connector; resource_type(syskeeper_proxy) -> emqx_bridge_syskeeper_proxy_server; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; -resource_type(timescale) -> - emqx_postgresql; resource_type(redis) -> emqx_bridge_redis_connector; resource_type(rocketmq) -> @@ -95,22 +87,6 @@ fields(connectors) -> connector_structs() -> [ - {kinesis, - mk( - hoconsc:map(name, ref(emqx_bridge_kinesis, "config_connector")), - #{ - desc => <<"Kinesis Connector Config">>, - required => false - } - )}, - {matrix, - mk( - hoconsc:map(name, ref(emqx_bridge_matrix, "config_connector")), - #{ - desc => <<"Matrix Connector Config">>, - required => false - } - )}, {mongodb, mk( hoconsc:map(name, ref(emqx_bridge_mongodb, "config_connector")), @@ -160,14 +136,6 @@ connector_structs() -> required => false } )}, - {pgsql, - mk( - hoconsc:map(name, ref(emqx_bridge_pgsql, "config_connector")), - #{ - desc => <<"PostgreSQL Connector Config">>, - required => false - } - )}, {redis, mk( hoconsc:map(name, ref(emqx_bridge_redis_schema, "config_connector")), @@ -208,14 +176,6 @@ connector_structs() -> required => false } )}, - {timescale, - mk( - hoconsc:map(name, ref(emqx_bridge_timescale, "config_connector")), - #{ - desc => <<"Timescale Connector Config">>, - required => false - } - )}, {iotdb, mk( hoconsc:map(name, ref(emqx_bridge_iotdb_connector, config)), @@ -284,8 +244,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_kinesis, - emqx_bridge_matrix, emqx_bridge_mongodb, emqx_bridge_oracle, emqx_bridge_influxdb, @@ -295,7 +253,6 @@ schema_modules() -> emqx_bridge_syskeeper_connector, emqx_bridge_syskeeper_proxy, emqx_bridge_sqlserver, - emqx_bridge_timescale, emqx_postgresql_connector_schema, emqx_bridge_redis_schema, emqx_bridge_rocketmq, @@ -313,8 +270,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_kinesis, <<"kinesis">>, Method ++ "_connector"), - api_ref(emqx_bridge_matrix, <<"matrix">>, Method ++ "_connector"), api_ref(emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector"), api_ref(emqx_bridge_oracle, <<"oracle">>, Method ++ "_connector"), api_ref(emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector"), @@ -324,8 +279,6 @@ api_schemas(Method) -> api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), - api_ref(emqx_bridge_timescale, <<"timescale">>, Method ++ "_connector"), - api_ref(emqx_postgresql_connector_schema, <<"pgsql">>, Method ++ "_connector"), api_ref(emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector"), api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method ++ "_connector"), api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 58308f8e3..4fae1d20c 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,10 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(kinesis) -> - [kinesis, kinesis_producer]; -connector_type_to_bridge_types(matrix) -> - [matrix]; connector_type_to_bridge_types(mongodb) -> [mongodb, mongodb_rs, mongodb_sharded, mongodb_single]; connector_type_to_bridge_types(oracle) -> @@ -145,8 +141,6 @@ connector_type_to_bridge_types(mysql) -> [mysql]; connector_type_to_bridge_types(mqtt) -> [mqtt]; -connector_type_to_bridge_types(pgsql) -> - [pgsql]; connector_type_to_bridge_types(redis) -> [redis, redis_single, redis_sentinel, redis_cluster]; connector_type_to_bridge_types(rocketmq) -> @@ -157,8 +151,6 @@ connector_type_to_bridge_types(syskeeper_proxy) -> []; connector_type_to_bridge_types(sqlserver) -> [sqlserver]; -connector_type_to_bridge_types(timescale) -> - [timescale]; connector_type_to_bridge_types(iotdb) -> [iotdb]; connector_type_to_bridge_types(elasticsearch) -> From df8c7152fc1a5c1727861bc8f2e5bb72b3a77cf4 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 19 Mar 2024 16:09:30 +0100 Subject: [PATCH 09/23] refactor(mongodb connector): to use emqx_connector_info This commit refactors the mongodb connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_mongodb.app.src | 5 ++- .../emqx_bridge_mongodb_connector_info.erl | 43 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector_info.erl diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index 372480fa6..2a18867f0 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -8,7 +8,10 @@ emqx_resource, emqx_mongodb ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_mongodb_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_mongodb_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_mongodb_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector_info.erl new file mode 100644 index 000000000..de69aacb5 --- /dev/null +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mongodb_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + mongodb. + +bridge_types() -> + [mongodb, mongodb_rs, mongodb_sharded, mongodb_single]. + +resource_callback_module() -> + emqx_bridge_mongodb_connector. + +config_schema() -> + {mongodb, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_mongodb, "config_connector")), + #{ + desc => <<"MongoDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_mongodb. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 538ecce58..fe5c8e514 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -67,7 +67,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_kinesis_connector_info, emqx_bridge_matrix_connector_info, emqx_bridge_pgsql_connector_info, - emqx_bridge_timescale_connector_info + emqx_bridge_timescale_connector_info, + emqx_bridge_mongodb_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 51c510806..022a2a0a8 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(mongodb) -> - emqx_bridge_mongodb_connector; resource_type(oracle) -> emqx_oracle; resource_type(influxdb) -> @@ -87,14 +85,6 @@ fields(connectors) -> connector_structs() -> [ - {mongodb, - mk( - hoconsc:map(name, ref(emqx_bridge_mongodb, "config_connector")), - #{ - desc => <<"MongoDB Connector Config">>, - required => false - } - )}, {oracle, mk( hoconsc:map(name, ref(emqx_bridge_oracle, "config_connector")), @@ -244,7 +234,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_mongodb, emqx_bridge_oracle, emqx_bridge_influxdb, emqx_bridge_cassandra, @@ -270,7 +259,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector"), api_ref(emqx_bridge_oracle, <<"oracle">>, Method ++ "_connector"), api_ref(emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector"), api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 4fae1d20c..9c0e7cc92 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(mongodb) -> - [mongodb, mongodb_rs, mongodb_sharded, mongodb_single]; connector_type_to_bridge_types(oracle) -> [oracle]; connector_type_to_bridge_types(influxdb) -> From c1e6092013602bcd5fd8765189f4282fe86422e0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 10:12:55 +0100 Subject: [PATCH 10/23] refactor(oracle connector): to use emqx_connector_info This commit refactors the oracle connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_oracle.app.src | 5 ++- .../src/emqx_bridge_oracle_connector_info.erl | 44 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 13 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_bridge_oracle/src/emqx_bridge_oracle_connector_info.erl diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src index 39b606d5f..1e360aa39 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -8,7 +8,10 @@ emqx_resource, emqx_oracle ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_oracle_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_oracle_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_oracle_connector_info]} + ]}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle_connector_info.erl b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle_connector_info.erl new file mode 100644 index 000000000..e4c3dcbf4 --- /dev/null +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle_connector_info.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_oracle_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + oracle. + +bridge_types() -> + [oracle]. + +resource_callback_module() -> + emqx_oracle. + +config_schema() -> + {oracle, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_oracle, "config_connector")), + #{ + desc => <<"Oracle Connector Config">>, + required => false, + validator => fun emqx_bridge_oracle:config_validator/1 + } + )}. + +schema_module() -> + emqx_bridge_oracle. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_oracle, <<"oracle">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index fe5c8e514..8f3ab7755 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -68,7 +68,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_matrix_connector_info, emqx_bridge_pgsql_connector_info, emqx_bridge_timescale_connector_info, - emqx_bridge_mongodb_connector_info + emqx_bridge_mongodb_connector_info, + emqx_bridge_oracle_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 022a2a0a8..b16864947 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(oracle) -> - emqx_oracle; resource_type(influxdb) -> emqx_bridge_influxdb_connector; resource_type(cassandra) -> @@ -85,15 +83,6 @@ fields(connectors) -> connector_structs() -> [ - {oracle, - mk( - hoconsc:map(name, ref(emqx_bridge_oracle, "config_connector")), - #{ - desc => <<"Oracle Connector Config">>, - required => false, - validator => fun emqx_bridge_oracle:config_validator/1 - } - )}, {influxdb, mk( hoconsc:map(name, ref(emqx_bridge_influxdb, "config_connector")), @@ -234,7 +223,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_oracle, emqx_bridge_influxdb, emqx_bridge_cassandra, emqx_bridge_clickhouse, @@ -259,7 +247,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_oracle, <<"oracle">>, Method ++ "_connector"), api_ref(emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector"), api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector"), api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 9c0e7cc92..2a0d92203 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(oracle) -> - [oracle]; connector_type_to_bridge_types(influxdb) -> [influxdb, influxdb_api_v1, influxdb_api_v2]; connector_type_to_bridge_types(cassandra) -> From 159ab8190444f5a9ece42a52f8d5fe86de9cbe3a Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 10:39:16 +0100 Subject: [PATCH 11/23] refactor(influxdb connector): to use emqx_connector_info This commit refactors the influxdb connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_influxdb.app.src | 7 +++- .../emqx_bridge_influxdb_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector_info.erl diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index de58c2170..d65a0bf11 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,10 @@ emqx_resource, influxdb ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_influxdb_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_influxdb_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_influxdb_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector_info.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector_info.erl new file mode 100644 index 000000000..db33bcf01 --- /dev/null +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_influxdb_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + influxdb. + +bridge_types() -> + [influxdb, influxdb_api_v1, influxdb_api_v2]. + +resource_callback_module() -> + emqx_bridge_influxdb_connector. + +config_schema() -> + {influxdb, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_influxdb, "config_connector")), + #{ + desc => <<"InfluxDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_influxdb. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 8f3ab7755..c5fb1d411 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -69,7 +69,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_pgsql_connector_info, emqx_bridge_timescale_connector_info, emqx_bridge_mongodb_connector_info, - emqx_bridge_oracle_connector_info + emqx_bridge_oracle_connector_info, + emqx_bridge_influxdb_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index b16864947..e46728188 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(influxdb) -> - emqx_bridge_influxdb_connector; resource_type(cassandra) -> emqx_bridge_cassandra_connector; resource_type(clickhouse) -> @@ -83,14 +81,6 @@ fields(connectors) -> connector_structs() -> [ - {influxdb, - mk( - hoconsc:map(name, ref(emqx_bridge_influxdb, "config_connector")), - #{ - desc => <<"InfluxDB Connector Config">>, - required => false - } - )}, {cassandra, mk( hoconsc:map(name, ref(emqx_bridge_cassandra, "config_connector")), @@ -223,7 +213,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_influxdb, emqx_bridge_cassandra, emqx_bridge_clickhouse, emqx_bridge_mysql, @@ -247,7 +236,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector"), api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector"), api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method ++ "_connector"), api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 2a0d92203..940d92cba 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(influxdb) -> - [influxdb, influxdb_api_v1, influxdb_api_v2]; connector_type_to_bridge_types(cassandra) -> [cassandra]; connector_type_to_bridge_types(clickhouse) -> From afa3da6d0d9740f652e153b9d4dd934e48c77212 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 10:50:44 +0100 Subject: [PATCH 12/23] refactor(cassandra connector): to use emqx_connector_info This commit refactors the cassandra connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_cassandra.app.src | 5 ++- .../src/emqx_bridge_cassandra_action_info.erl | 3 ++ .../emqx_bridge_cassandra_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 6 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector_info.erl diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index aa8290b98..84f9ccd21 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -8,7 +8,10 @@ emqx_resource, ecql ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_cassandra_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_cassandra_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_cassandra_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl index 14db7cf50..158207971 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl @@ -1,3 +1,6 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(emqx_bridge_cassandra_action_info). -behaviour(emqx_action_info). diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector_info.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector_info.erl new file mode 100644 index 000000000..b86740625 --- /dev/null +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_cassandra_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + cassandra. + +bridge_types() -> + [cassandra]. + +resource_callback_module() -> + emqx_bridge_cassandra_connector. + +config_schema() -> + {cassandra, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_cassandra, "config_connector")), + #{ + desc => <<"Cassandra Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_cassandra. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index c5fb1d411..568c66ec4 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -70,7 +70,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_timescale_connector_info, emqx_bridge_mongodb_connector_info, emqx_bridge_oracle_connector_info, - emqx_bridge_influxdb_connector_info + emqx_bridge_influxdb_connector_info, + emqx_bridge_cassandra_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index e46728188..98b55606d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(cassandra) -> - emqx_bridge_cassandra_connector; resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(mysql) -> @@ -81,14 +79,6 @@ fields(connectors) -> connector_structs() -> [ - {cassandra, - mk( - hoconsc:map(name, ref(emqx_bridge_cassandra, "config_connector")), - #{ - desc => <<"Cassandra Connector Config">>, - required => false - } - )}, {clickhouse, mk( hoconsc:map(name, ref(emqx_bridge_clickhouse, "config_connector")), @@ -213,7 +203,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_cassandra, emqx_bridge_clickhouse, emqx_bridge_mysql, emqx_bridge_syskeeper_connector, @@ -236,7 +225,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector"), api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method ++ "_connector"), api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 940d92cba..8236f57aa 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(cassandra) -> - [cassandra]; connector_type_to_bridge_types(clickhouse) -> [clickhouse]; connector_type_to_bridge_types(mysql) -> From c6d7c57bbd3358f984b202a6b1b617951ddd5cb9 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 10:59:58 +0100 Subject: [PATCH 13/23] refactor(clickhouse connector): to use emqx_connector_info This commit refactors the clickhouse connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_clickhouse.app.src | 5 ++- .../emqx_bridge_clickhouse_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector_info.erl diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index d96d06375..b0fc2698c 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -8,7 +8,10 @@ emqx_resource, clickhouse ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_clickhouse_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_clickhouse_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_clickhouse_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector_info.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector_info.erl new file mode 100644 index 000000000..8aa03b27e --- /dev/null +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_clickhouse_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + clickhouse. + +bridge_types() -> + [clickhouse]. + +resource_callback_module() -> + emqx_bridge_clickhouse_connector. + +config_schema() -> + {clickhouse, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_clickhouse, "config_connector")), + #{ + desc => <<"ClickHouse Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_clickhouse. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_clickhouse, <<"clickhouse">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 568c66ec4..2ff4191fa 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -71,7 +71,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_mongodb_connector_info, emqx_bridge_oracle_connector_info, emqx_bridge_influxdb_connector_info, - emqx_bridge_cassandra_connector_info + emqx_bridge_cassandra_connector_info, + emqx_bridge_clickhouse_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 98b55606d..aed05d96a 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(clickhouse) -> - emqx_bridge_clickhouse_connector; resource_type(mysql) -> emqx_bridge_mysql_connector; resource_type(syskeeper_forwarder) -> @@ -79,14 +77,6 @@ fields(connectors) -> connector_structs() -> [ - {clickhouse, - mk( - hoconsc:map(name, ref(emqx_bridge_clickhouse, "config_connector")), - #{ - desc => <<"ClickHouse Connector Config">>, - required => false - } - )}, {mysql, mk( hoconsc:map(name, ref(emqx_bridge_mysql, "config_connector")), @@ -203,7 +193,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_clickhouse, emqx_bridge_mysql, emqx_bridge_syskeeper_connector, emqx_bridge_syskeeper_proxy, @@ -225,7 +214,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method ++ "_connector"), api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 8236f57aa..fabbad32b 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(clickhouse) -> - [clickhouse]; connector_type_to_bridge_types(mysql) -> [mysql]; connector_type_to_bridge_types(mqtt) -> From 223bbc05fea73844f1ba92dce1eb11f648f2f8f4 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 11:12:38 +0100 Subject: [PATCH 14/23] refactor(mysql connector): to use emqx_connector_info This commit refactors the mysql connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_mysql.app.src | 7 +++- .../src/emqx_bridge_mysql_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector_info.erl diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index be5f4b417..f02600336 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,10 @@ emqx_resource, emqx_mysql ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_mysql_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_mysql_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_mysql_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector_info.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector_info.erl new file mode 100644 index 000000000..c55e7e878 --- /dev/null +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mysql_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + mysql. + +bridge_types() -> + [mysql]. + +resource_callback_module() -> + emqx_bridge_mysql_connector. + +config_schema() -> + {mysql, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_mysql, "config_connector")), + #{ + desc => <<"MySQL Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_mysql. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 2ff4191fa..c44c2b18c 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -72,7 +72,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_oracle_connector_info, emqx_bridge_influxdb_connector_info, emqx_bridge_cassandra_connector_info, - emqx_bridge_clickhouse_connector_info + emqx_bridge_clickhouse_connector_info, + emqx_bridge_mysql_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index aed05d96a..57c176b02 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,8 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(mysql) -> - emqx_bridge_mysql_connector; resource_type(syskeeper_forwarder) -> emqx_bridge_syskeeper_connector; resource_type(syskeeper_proxy) -> @@ -77,14 +75,6 @@ fields(connectors) -> connector_structs() -> [ - {mysql, - mk( - hoconsc:map(name, ref(emqx_bridge_mysql, "config_connector")), - #{ - desc => <<"MySQL Connector Config">>, - required => false - } - )}, {redis, mk( hoconsc:map(name, ref(emqx_bridge_redis_schema, "config_connector")), @@ -193,7 +183,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_mysql, emqx_bridge_syskeeper_connector, emqx_bridge_syskeeper_proxy, emqx_bridge_sqlserver, @@ -214,7 +203,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index fabbad32b..01b6ec925 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -127,8 +127,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(mysql) -> - [mysql]; connector_type_to_bridge_types(mqtt) -> [mqtt]; connector_type_to_bridge_types(redis) -> From 12717baa0b8594782a4fa590c15dcaf089f8277a Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 13:33:13 +0100 Subject: [PATCH 15/23] refactor(mqtt connector): to use emqx_connector_info This commit refactors the mqtt connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_mqtt.app.src | 5 +- ...emqx_bridge_mqtt_pubsub_connector_info.erl | 55 +++++++++++++++++++ .../src/emqx_connector_info.erl | 5 +- .../src/emqx_connector_resource.erl | 4 -- .../src/schema/emqx_connector_schema.erl | 24 ++------ 5 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_connector_info.erl diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 6fbc0edde..52c69b21b 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -10,7 +10,10 @@ emqx_resource, emqtt ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_mqtt_pubsub_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_mqtt_pubsub_connector_info]} + ]}, {modules, []}, {licenses, ["Apache 2.0"]}, {links, []} diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_connector_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_connector_info.erl new file mode 100644 index 000000000..9b64428cc --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_connector_info.erl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_pubsub_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + mqtt. + +bridge_types() -> + [mqtt]. + +resource_callback_module() -> + emqx_bridge_mqtt_connector. + +config_schema() -> + {mqtt, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + #{ + desc => <<"MQTT Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_mqtt_connector_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index c44c2b18c..67de571f9 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -81,7 +81,10 @@ hard_coded_connector_info_modules_ee() -> -endif. hard_coded_connector_info_modules_common() -> - [emqx_bridge_http_connector_info]. + [ + emqx_bridge_http_connector_info, + emqx_bridge_mqtt_pubsub_connector_info + ]. hard_coded_connector_info_modules() -> hard_coded_connector_info_modules_common() ++ hard_coded_connector_info_modules_ee(). diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index dec4ab2d7..9b109b732 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -88,10 +88,6 @@ connector_impl_module(ConnectorType) -> -endif. -connector_to_resource_type_ce(mqtt) -> - emqx_bridge_mqtt_connector; -% connector_to_resource_type_ce(mqtt_subscriber) -> -% emqx_bridge_mqtt_subscriber_connector; connector_to_resource_type_ce(ConnectorType) -> try emqx_connector_info:resource_callback_module(ConnectorType) diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 01b6ec925..2af7cc52d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -88,11 +88,7 @@ enterprise_fields_connectors() -> []. -endif. api_schemas(Method) -> - [ - %% We need to map the `type' field of a request (binary) to a - %% connector schema module. - api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector") - ]. + []. api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. @@ -111,11 +107,10 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - [emqx_bridge_mqtt_connector_schema] ++ - emqx_connector_ee_schema:schema_modules() ++ connector_info_schema_modules(). + emqx_connector_ee_schema:schema_modules() ++ connector_info_schema_modules(). -else. schema_modules() -> - [emqx_bridge_mqtt_connector_schema] ++ connector_info_schema_modules(). + connector_info_schema_modules(). -endif. connector_info_schema_modules() -> @@ -127,8 +122,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(mqtt) -> - [mqtt]; connector_type_to_bridge_types(redis) -> [redis, redis_single, redis_sentinel, redis_cluster]; connector_type_to_bridge_types(rocketmq) -> @@ -505,16 +498,7 @@ roots() -> end. fields(connectors) -> - [ - {mqtt, - mk( - hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), - #{ - desc => <<"MQTT Publisher Connector Config">>, - required => false - } - )} - ] ++ enterprise_fields_connectors() ++ connector_info_fields_connectors(); + [] ++ enterprise_fields_connectors() ++ connector_info_fields_connectors(); fields("node_status") -> [ node_name(), From aa368bb9d04b4a34054f26ce964e3b643fc50923 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 13:49:28 +0100 Subject: [PATCH 16/23] refactor(redis connector): to use emqx_connector_info This commit refactors the redis connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_redis.app.src | 5 ++- .../src/emqx_bridge_redis_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 4 +- 5 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 8e737c4a3..57b4d90f1 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, @@ -9,7 +9,8 @@ emqx_redis ]}, {env, [ - {emqx_action_info_modules, [emqx_bridge_redis_action_info]} + {emqx_action_info_modules, [emqx_bridge_redis_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_redis_connector_info]} ]}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl new file mode 100644 index 000000000..8a3cdd6cd --- /dev/null +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_redis_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + redis. + +bridge_types() -> + [redis, redis_single, redis_sentinel, redis_cluster]. + +resource_callback_module() -> + emqx_bridge_redis_connector. + +config_schema() -> + {redis, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_redis_schema, "config_connector")), + #{ + desc => <<"Redis Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_redis_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_redis_schema, <<"mysql">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 67de571f9..36b01d6eb 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -73,7 +73,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_influxdb_connector_info, emqx_bridge_cassandra_connector_info, emqx_bridge_clickhouse_connector_info, - emqx_bridge_mysql_connector_info + emqx_bridge_mysql_connector_info, + emqx_bridge_redis_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 57c176b02..f2aed272d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -27,8 +27,6 @@ resource_type(syskeeper_proxy) -> emqx_bridge_syskeeper_proxy_server; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; -resource_type(redis) -> - emqx_bridge_redis_connector; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(iotdb) -> @@ -75,14 +73,6 @@ fields(connectors) -> connector_structs() -> [ - {redis, - mk( - hoconsc:map(name, ref(emqx_bridge_redis_schema, "config_connector")), - #{ - desc => <<"Redis Connector Config">>, - required => false - } - )}, {rocketmq, mk( hoconsc:map(name, ref(emqx_bridge_rocketmq, "config_connector")), @@ -187,7 +177,6 @@ schema_modules() -> emqx_bridge_syskeeper_proxy, emqx_bridge_sqlserver, emqx_postgresql_connector_schema, - emqx_bridge_redis_schema, emqx_bridge_rocketmq, emqx_bridge_iotdb_connector, emqx_bridge_es_connector, @@ -206,7 +195,6 @@ api_schemas(Method) -> api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), - api_ref(emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector"), api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method ++ "_connector"), api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 2af7cc52d..0a1967dd1 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -87,7 +87,7 @@ enterprise_fields_connectors() -> []. -endif. -api_schemas(Method) -> +api_schemas(_Method) -> []. api_ref(Module, Type, Method) -> @@ -122,8 +122,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(redis) -> - [redis, redis_single, redis_sentinel, redis_cluster]; connector_type_to_bridge_types(rocketmq) -> [rocketmq]; connector_type_to_bridge_types(syskeeper_forwarder) -> From 188ab716733685dc6131b7aba1a2bebf4db058e0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 14:02:18 +0100 Subject: [PATCH 17/23] refactor(rocketmq connector): to use emqx_connector_info This commit refactors the rocketmq connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_rocketmq.app.src | 5 ++- .../emqx_bridge_rocketmq_connector_info.erl | 42 +++++++++++++++++++ .../src/emqx_connector_info.erl | 3 +- .../src/schema/emqx_connector_ee_schema.erl | 12 ------ .../src/schema/emqx_connector_schema.erl | 2 - 5 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector_info.erl diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index 564e36a88..f41b313d4 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -3,7 +3,10 @@ {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, rocketmq]}, - {env, [{emqx_action_info_modules, [emqx_bridge_rocketmq_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_rocketmq_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_rocketmq_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector_info.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector_info.erl new file mode 100644 index 000000000..96783f53c --- /dev/null +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector_info.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_rocketmq_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + rocketmq. + +bridge_types() -> + [rocketmq]. + +resource_callback_module() -> + emqx_bridge_rocketmq_connector. + +config_schema() -> + {rocketmq, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_rocketmq, "config_connector")), + #{ + desc => <<"RocketMQ Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_rocketmq. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_rocketmq, <<"rocketmq">>, Method ++ "_connector" + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 36b01d6eb..18f952733 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -74,7 +74,8 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_cassandra_connector_info, emqx_bridge_clickhouse_connector_info, emqx_bridge_mysql_connector_info, - emqx_bridge_redis_connector_info + emqx_bridge_redis_connector_info, + emqx_bridge_rocketmq_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index f2aed272d..60c94625d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -27,8 +27,6 @@ resource_type(syskeeper_proxy) -> emqx_bridge_syskeeper_proxy_server; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; -resource_type(rocketmq) -> - emqx_bridge_rocketmq_connector; resource_type(iotdb) -> emqx_bridge_iotdb_connector; resource_type(elasticsearch) -> @@ -73,14 +71,6 @@ fields(connectors) -> connector_structs() -> [ - {rocketmq, - mk( - hoconsc:map(name, ref(emqx_bridge_rocketmq, "config_connector")), - #{ - desc => <<"RocketMQ Connector Config">>, - required => false - } - )}, {syskeeper_forwarder, mk( hoconsc:map(name, ref(emqx_bridge_syskeeper_connector, config)), @@ -177,7 +167,6 @@ schema_modules() -> emqx_bridge_syskeeper_proxy, emqx_bridge_sqlserver, emqx_postgresql_connector_schema, - emqx_bridge_rocketmq, emqx_bridge_iotdb_connector, emqx_bridge_es_connector, emqx_bridge_rabbitmq_connector_schema, @@ -195,7 +184,6 @@ api_schemas(Method) -> api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), - api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method ++ "_connector"), api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method), api_ref(emqx_bridge_opents_connector, <<"opents">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 0a1967dd1..327e305e4 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -122,8 +122,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(rocketmq) -> - [rocketmq]; connector_type_to_bridge_types(syskeeper_forwarder) -> [syskeeper_forwarder]; connector_type_to_bridge_types(syskeeper_proxy) -> From 27d109ce6d4adca03b4d1ca02bfb32fbfbbbccc3 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 14:29:02 +0100 Subject: [PATCH 18/23] refactor(syskeeper connector): to use emqx_connector_info This commit refactors the syskeeper connector to use the `emqx_connector_info` behavior. --- .../src/emqx_bridge_syskeeper.app.src | 10 +++- .../emqx_bridge_syskeeper_connector_info.erl | 46 ++++++++++++++++++ ..._bridge_syskeeper_proxy_connector_info.erl | 47 +++++++++++++++++++ .../src/emqx_connector_info.erl | 4 +- .../src/schema/emqx_connector_ee_schema.erl | 24 ---------- .../src/schema/emqx_connector_schema.erl | 4 -- 6 files changed, 104 insertions(+), 31 deletions(-) create mode 100644 apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector_info.erl create mode 100644 apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_connector_info.erl diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src index 8ac88b9bd..4f461ab4e 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -1,13 +1,19 @@ {application, emqx_bridge_syskeeper, [ {description, "EMQX Enterprise Data bridge for Syskeeper"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, stdlib, emqx_resource ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_syskeeper_action_info]}, + {emqx_connector_info_modules, [ + emqx_bridge_syskeeper_connector_info, + emqx_bridge_syskeeper_proxy_connector_info + ]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector_info.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector_info.erl new file mode 100644 index 000000000..5dac5fcdd --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector_info.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_syskeeper_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + syskeeper_forwarder. + +bridge_types() -> + [syskeeper_forwarder]. + +resource_callback_module() -> + emqx_bridge_syskeeper_connector. + +config_transform_module() -> + emqx_bridge_syskeeper_connector. + +config_schema() -> + {syskeeper_forwarder, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_syskeeper_connector, config)), + #{ + desc => <<"Syskeeper Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_syskeeper_connector. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method + ). diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_connector_info.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_connector_info.erl new file mode 100644 index 000000000..cd0bf2428 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_proxy_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + syskeeper_proxy. + +bridge_types() -> + []. + +resource_callback_module() -> + emqx_bridge_syskeeper_proxy_server. + +config_transform_module() -> + emqx_bridge_syskeeper_proxy. + +config_schema() -> + {syskeeper_proxy, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_syskeeper_proxy, config)), + #{ + desc => <<"Syskeeper Proxy Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_syskeeper_proxy. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 18f952733..bc80ed6c7 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -75,7 +75,9 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_clickhouse_connector_info, emqx_bridge_mysql_connector_info, emqx_bridge_redis_connector_info, - emqx_bridge_rocketmq_connector_info + emqx_bridge_rocketmq_connector_info, + emqx_bridge_syskeeper_connector_info, + emqx_bridge_syskeeper_proxy_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 60c94625d..0db605bfe 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,10 +21,6 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(syskeeper_forwarder) -> - emqx_bridge_syskeeper_connector; -resource_type(syskeeper_proxy) -> - emqx_bridge_syskeeper_proxy_server; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; resource_type(iotdb) -> @@ -71,22 +67,6 @@ fields(connectors) -> connector_structs() -> [ - {syskeeper_forwarder, - mk( - hoconsc:map(name, ref(emqx_bridge_syskeeper_connector, config)), - #{ - desc => <<"Syskeeper Connector Config">>, - required => false - } - )}, - {syskeeper_proxy, - mk( - hoconsc:map(name, ref(emqx_bridge_syskeeper_proxy, config)), - #{ - desc => <<"Syskeeper Proxy Connector Config">>, - required => false - } - )}, {sqlserver, mk( hoconsc:map(name, ref(emqx_bridge_sqlserver, "config_connector")), @@ -163,8 +143,6 @@ connector_structs() -> schema_modules() -> [ - emqx_bridge_syskeeper_connector, - emqx_bridge_syskeeper_proxy, emqx_bridge_sqlserver, emqx_postgresql_connector_schema, emqx_bridge_iotdb_connector, @@ -181,8 +159,6 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), - api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 327e305e4..529de41a0 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -122,10 +122,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(syskeeper_forwarder) -> - [syskeeper_forwarder]; -connector_type_to_bridge_types(syskeeper_proxy) -> - []; connector_type_to_bridge_types(sqlserver) -> [sqlserver]; connector_type_to_bridge_types(iotdb) -> From a3b75eb39de9d752abbc2414f9d03bec75cdb0e6 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 20 Mar 2024 16:16:32 +0100 Subject: [PATCH 19/23] refactor(remaining connectors): to use emqx_connector_info This commit refactors the sqlserver, iotdb, elasticsearch, opents, greptimedb, pulsar, tdengine, rabbitmq and s3 connectors to use the `emqx_connector_info` behavior. --- .../emqx_bridge_es/src/emqx_bridge_es.app.src | 7 +- .../src/emqx_bridge_es_connector_info.erl | 43 ++++++ .../src/emqx_bridge_greptimedb.app.src | 5 +- .../emqx_bridge_greptimedb_connector_info.erl | 43 ++++++ .../src/emqx_bridge_iotdb.app.src | 5 +- .../src/emqx_bridge_iotdb_connector_info.erl | 47 ++++++ .../src/emqx_bridge_opents.app.src | 5 +- .../src/emqx_bridge_opents_connector_info.erl | 47 ++++++ .../src/emqx_bridge_pulsar.app.src | 8 +- .../src/emqx_bridge_pulsar_connector_info.erl | 47 ++++++ .../src/emqx_bridge_rabbitmq.app.src | 6 +- .../emqx_bridge_rabbitmq_connector_info.erl | 49 +++++++ .../src/emqx_bridge_redis_connector_info.erl | 2 +- .../emqx_bridge_s3/src/emqx_bridge_s3.app.src | 5 +- .../src/emqx_bridge_s3_connector_info.erl | 47 ++++++ .../src/emqx_bridge_sqlserver.app.src | 5 +- .../emqx_bridge_sqlserver_connector_info.erl | 47 ++++++ .../src/emqx_bridge_tdengine.app.src | 5 +- .../emqx_bridge_tdengine_connector_info.erl | 47 ++++++ .../src/emqx_connector_info.erl | 11 +- .../src/schema/emqx_connector_ee_schema.erl | 137 +----------------- .../src/schema/emqx_connector_schema.erl | 18 --- 22 files changed, 471 insertions(+), 165 deletions(-) create mode 100644 apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl create mode 100644 apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector_info.erl create mode 100644 apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector_info.erl create mode 100644 apps/emqx_bridge_opents/src/emqx_bridge_opents_connector_info.erl create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_info.erl create mode 100644 apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_info.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl create mode 100644 apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector_info.erl create mode 100644 apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector_info.erl diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 34b07d305..9ed9d572d 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_es, [ {description, "EMQX Enterprise Elastic Search Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {modules, [ emqx_bridge_es, emqx_bridge_es_connector @@ -12,7 +12,10 @@ stdlib, emqx_resource ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_es_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_es_connector_info]} + ]}, {licenses, ["Business Source License 1.1"]}, {maintainers, ["EMQX Team "]}, {links, [ diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl new file mode 100644 index 000000000..4542c9382 --- /dev/null +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl @@ -0,0 +1,43 @@ +-module(emqx_bridge_es_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + elasticsearch. + +bridge_types() -> + [elasticsearch]. + +resource_callback_module() -> + emqx_bridge_es_connector. + +config_transform_module() -> + emqx_bridge_es_connector. + +config_schema() -> + {elasticsearch, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_es_connector, config)), + #{ + desc => <<"ElasticSearch Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_azure_event_hub. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_es_connector, <<"elasticsearch">>, Method + ). diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index 357db5c96..c61335f8f 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -8,7 +8,10 @@ emqx_resource, greptimedb ]}, - {env, [{emqx_action_info_modules, [emqx_bridge_greptimedb_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_greptimedb_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_greptimedb_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector_info.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector_info.erl new file mode 100644 index 000000000..cf804c45a --- /dev/null +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector_info.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_greptimedb_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + greptimedb. + +bridge_types() -> + [greptimedb]. + +resource_callback_module() -> + emqx_bridge_greptimedb_connector. + +config_schema() -> + {greptimedb, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_greptimedb, "config_connector")), + #{ + desc => <<"GreptimeDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_greptimedb. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_greptimedb, <<"greptimedb">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index b666beeed..9515d8d19 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -12,7 +12,10 @@ stdlib, emqx_resource ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_iotdb_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_iotdb_connector_info]} + ]}, {licenses, ["Business Source License 1.1"]}, {maintainers, ["EMQX Team "]}, {links, [ diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector_info.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector_info.erl new file mode 100644 index 000000000..f91151d87 --- /dev/null +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_iotdb_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + iotdb. + +bridge_types() -> + [iotdb]. + +resource_callback_module() -> + emqx_bridge_iotdb_connector. + +config_transform_module() -> + emqx_bridge_iotdb_connector. + +config_schema() -> + {iotdb, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_iotdb_connector, config)), + #{ + desc => <<"IoTDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_iotdb_connector. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_iotdb_connector, <<"iotdb">>, Method + ). diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 2469acaa8..8514eb37c 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -8,7 +8,10 @@ emqx_resource, opentsdb ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_opents_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_opents_connector_info]} + ]}, {modules, []}, {licenses, ["BSL"]}, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector_info.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector_info.erl new file mode 100644 index 000000000..e312e7d4a --- /dev/null +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_opents_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + opents. + +bridge_types() -> + [opents]. + +resource_callback_module() -> + emqx_bridge_opents_connector. + +config_transform_module() -> + emqx_bridge_opents_connector. + +config_schema() -> + {opents, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_opents_connector, "config_connector")), + #{ + desc => <<"OpenTSDB Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_opents_connector. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_opents_connector, <<"opents">>, Method + ). diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index ce7c313ae..dc61ae334 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,11 @@ emqx_resource, pulsar ]}, - {env, []}, + + {env, [ + {emqx_action_info_modules, [emqx_bridge_pulsar_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_pulsar_connector_info]} + ]}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_info.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_info.erl new file mode 100644 index 000000000..94f8be03d --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_pulsar_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + pulsar. + +bridge_types() -> + [pulsar_producer, pulsar]. + +resource_callback_module() -> + emqx_bridge_pulsar_connector. + +config_transform_module() -> + emqx_bridge_pulsar_connector. + +config_schema() -> + {pulsar, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_pulsar_connector_schema, "config_connector")), + #{ + desc => <<"Pulsar Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_pulsar_connector_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_pulsar_connector_schema, <<"pulsar">>, Method + ). diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index a885cc6bc..b9e271bf9 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -10,7 +10,11 @@ rabbit_common, amqp_client ]}, - {env, []}, + + {env, [ + {emqx_action_info_modules, [emqx_bridge_rabbitmq_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_rabbitmq_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_info.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_info.erl new file mode 100644 index 000000000..9a48edae7 --- /dev/null +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_info.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_rabbitmq_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + rabbitmq. + +bridge_types() -> + [rabbitmq]. + +resource_callback_module() -> + emqx_bridge_rabbitmq_connector. + +config_transform_module() -> + emqx_bridge_rabbitmq_connector. + +config_schema() -> + {rabbitmq, + hoconsc:mk( + hoconsc:map( + name, hoconsc:ref(emqx_bridge_rabbitmq_connector_schema, "config_connector") + ), + #{ + desc => <<"RabbitMQ Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_rabbitmq_connector_schema. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_rabbitmq_connector_schema, <<"rabbitmq">>, Method + ). diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl index 8a3cdd6cd..3de7e8108 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector_info.erl @@ -38,5 +38,5 @@ schema_module() -> api_schema(Method) -> emqx_connector_schema:api_ref( - emqx_bridge_redis_schema, <<"mysql">>, Method ++ "_connector" + emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector" ). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src index 0047b5e51..a6000067a 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_s3, [ {description, "EMQX Enterprise S3 Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, @@ -10,7 +10,8 @@ emqx_s3 ]}, {env, [ - {emqx_action_info_modules, [emqx_bridge_s3_action_info]} + {emqx_action_info_modules, [emqx_bridge_s3_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_s3_connector_info]} ]}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl new file mode 100644 index 000000000..be3c29bae --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + s3. + +bridge_types() -> + [s3]. + +resource_callback_module() -> + emqx_bridge_s3_connector. + +config_transform_module() -> + emqx_bridge_s3. + +config_schema() -> + {s3, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_s3, "config_connector")), + #{ + desc => <<"S3 Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_s3. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_s3, <<"s3">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 1340161a2..5931b17de 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -3,7 +3,10 @@ {vsn, "0.1.6"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, odbc]}, - {env, [{emqx_action_info_modules, [emqx_bridge_sqlserver_action_info]}]}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_sqlserver_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_sqlserver_connector_info]} + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector_info.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector_info.erl new file mode 100644 index 000000000..8dbeaa30b --- /dev/null +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_sqlserver_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + sqlserver. + +bridge_types() -> + [sqlserver]. + +resource_callback_module() -> + emqx_bridge_sqlserver_connector. + +config_transform_module() -> + emqx_bridge_sqlserver. + +config_schema() -> + {sqlserver, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_sqlserver, "config_connector")), + #{ + desc => <<"Microsoft SQL Server Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_sqlserver. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector" + ). diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 898a3211d..b1c3c2d6a 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -8,7 +8,10 @@ emqx_resource, tdengine ]}, - {env, []}, + {env, [ + {emqx_action_info_modules, [emqx_bridge_tdengine_action_info]}, + {emqx_connector_info_modules, [emqx_bridge_tdengine_connector_info]} + ]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector_info.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector_info.erl new file mode 100644 index 000000000..010e77269 --- /dev/null +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector_info.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_tdengine_connector_info). + +-behaviour(emqx_connector_info). + +-export([ + type_name/0, + bridge_types/0, + resource_callback_module/0, + config_transform_module/0, + config_schema/0, + schema_module/0, + api_schema/1 +]). + +type_name() -> + tdengine. + +bridge_types() -> + [tdengine]. + +resource_callback_module() -> + emqx_bridge_tdengine_connector. + +config_transform_module() -> + emqx_bridge_tdengine_connector. + +config_schema() -> + {tdengine, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(emqx_bridge_tdengine_connector, "config_connector")), + #{ + desc => <<"TDengine Connector Config">>, + required => false + } + )}. + +schema_module() -> + emqx_bridge_tdengine_connector. + +api_schema(Method) -> + emqx_connector_schema:api_ref( + emqx_bridge_tdengine_connector, <<"tdengine">>, Method + ). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index bc80ed6c7..c53fede62 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -77,7 +77,16 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_redis_connector_info, emqx_bridge_rocketmq_connector_info, emqx_bridge_syskeeper_connector_info, - emqx_bridge_syskeeper_proxy_connector_info + emqx_bridge_syskeeper_proxy_connector_info, + emqx_bridge_sqlserver_connector_info, + emqx_bridge_iotdb_connector_info, + emqx_bridge_es_connector_info, + emqx_bridge_opents_connector_info, + emqx_bridge_greptimedb_connector_info, + emqx_bridge_pulsar_connector_info, + emqx_bridge_tdengine_connector_info, + emqx_bridge_rabbitmq_connector_info, + emqx_bridge_s3_connector_info ]. -else. hard_coded_connector_info_modules_ee() -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 0db605bfe..8e5f1c001 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -21,42 +21,12 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(sqlserver) -> - emqx_bridge_sqlserver_connector; -resource_type(iotdb) -> - emqx_bridge_iotdb_connector; -resource_type(elasticsearch) -> - emqx_bridge_es_connector; -resource_type(opents) -> - emqx_bridge_opents_connector; -resource_type(greptimedb) -> - emqx_bridge_greptimedb_connector; -resource_type(tdengine) -> - emqx_bridge_tdengine_connector; -resource_type(pulsar) -> - emqx_bridge_pulsar_connector; -resource_type(rabbitmq) -> - emqx_bridge_rabbitmq_connector; -resource_type(s3) -> - emqx_bridge_s3_connector; resource_type(Type) -> error({unknown_connector_type, Type}). %% For connectors that need to override connector configurations. connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(iotdb) -> - emqx_bridge_iotdb_connector; -connector_impl_module(elasticsearch) -> - emqx_bridge_es_connector; -connector_impl_module(opents) -> - emqx_bridge_opents_connector; -connector_impl_module(pulsar) -> - emqx_bridge_pulsar_connector; -connector_impl_module(tdengine) -> - emqx_bridge_tdengine_connector; -connector_impl_module(rabbitmq) -> - emqx_bridge_rabbitmq_connector; connector_impl_module(_ConnectorType) -> undefined. @@ -66,112 +36,13 @@ fields(connectors) -> connector_structs(). connector_structs() -> - [ - {sqlserver, - mk( - hoconsc:map(name, ref(emqx_bridge_sqlserver, "config_connector")), - #{ - desc => <<"Microsoft SQL Server Connector Config">>, - required => false - } - )}, - {iotdb, - mk( - hoconsc:map(name, ref(emqx_bridge_iotdb_connector, config)), - #{ - desc => <<"IoTDB Connector Config">>, - required => false - } - )}, - {elasticsearch, - mk( - hoconsc:map(name, ref(emqx_bridge_es_connector, config)), - #{ - desc => <<"ElasticSearch Connector Config">>, - required => false - } - )}, - {opents, - mk( - hoconsc:map(name, ref(emqx_bridge_opents_connector, "config_connector")), - #{ - desc => <<"OpenTSDB Connector Config">>, - required => false - } - )}, - {greptimedb, - mk( - hoconsc:map(name, ref(emqx_bridge_greptimedb, "config_connector")), - #{ - desc => <<"GreptimeDB Connector Config">>, - required => false - } - )}, - {tdengine, - mk( - hoconsc:map(name, ref(emqx_bridge_tdengine_connector, "config_connector")), - #{ - desc => <<"TDengine Connector Config">>, - required => false - } - )}, - {pulsar, - mk( - hoconsc:map(name, ref(emqx_bridge_pulsar_connector_schema, "config_connector")), - #{ - desc => <<"Pulsar Connector Config">>, - required => false - } - )}, - {rabbitmq, - mk( - hoconsc:map(name, ref(emqx_bridge_rabbitmq_connector_schema, "config_connector")), - #{ - desc => <<"RabbitMQ Connector Config">>, - required => false - } - )}, - {s3, - mk( - hoconsc:map(name, ref(emqx_bridge_s3, "config_connector")), - #{ - desc => <<"S3 Connector Config">>, - required => false - } - )} - ]. + []. schema_modules() -> - [ - emqx_bridge_sqlserver, - emqx_postgresql_connector_schema, - emqx_bridge_iotdb_connector, - emqx_bridge_es_connector, - emqx_bridge_rabbitmq_connector_schema, - emqx_bridge_pulsar_connector_schema, - emqx_bridge_opents_connector, - emqx_bridge_greptimedb, - emqx_bridge_tdengine_connector, - emqx_bridge_s3 - ]. + []. -api_schemas(Method) -> - [ - %% We need to map the `type' field of a request (binary) to a - %% connector schema module. - api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"), - api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), - api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method), - api_ref(emqx_bridge_opents_connector, <<"opents">>, Method), - api_ref(emqx_bridge_rabbitmq_connector_schema, <<"rabbitmq">>, Method), - api_ref(emqx_bridge_pulsar_connector_schema, <<"pulsar">>, Method), - api_ref(emqx_bridge_greptimedb, <<"greptimedb">>, Method ++ "_connector"), - api_ref(emqx_bridge_tdengine_connector, <<"tdengine">>, Method), - api_ref(emqx_bridge_s3, <<"s3">>, Method ++ "_connector") - ]. - -api_ref(Module, Type, Method) -> - {Type, ref(Module, Method)}. +api_schemas(_Method) -> + []. -else. diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 529de41a0..7bfcab254 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -122,24 +122,6 @@ connector_info_schema_modules() -> %% @doc Return old bridge(v1) and/or connector(v2) type %% from the latest connector type name. -connector_type_to_bridge_types(sqlserver) -> - [sqlserver]; -connector_type_to_bridge_types(iotdb) -> - [iotdb]; -connector_type_to_bridge_types(elasticsearch) -> - [elasticsearch]; -connector_type_to_bridge_types(opents) -> - [opents]; -connector_type_to_bridge_types(greptimedb) -> - [greptimedb]; -connector_type_to_bridge_types(pulsar) -> - [pulsar_producer, pulsar]; -connector_type_to_bridge_types(tdengine) -> - [tdengine]; -connector_type_to_bridge_types(rabbitmq) -> - [rabbitmq]; -connector_type_to_bridge_types(s3) -> - [s3]; connector_type_to_bridge_types(Type) -> emqx_connector_info:bridge_types(Type). From 89befa580eebdd3182345df396603b13139ec282 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 21 Mar 2024 11:58:54 +0100 Subject: [PATCH 20/23] refactor(emqx_connector_info): remove code that is no longer needed --- .../test/emqx_bridge_v2_api_SUITE.erl | 12 +---- .../src/emqx_connector_resource.erl | 36 ++------------ .../src/schema/emqx_connector_ee_schema.erl | 49 ------------------- .../src/schema/emqx_connector_schema.erl | 46 +---------------- .../test/emqx_connector_SUITE.erl | 22 ++++----- .../test/emqx_connector_api_SUITE.erl | 6 +-- 6 files changed, 23 insertions(+), 148 deletions(-) delete mode 100644 apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 3fb85ae1c..b1e1ac38d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -248,16 +248,8 @@ end_per_testcase(_TestCase, Config) -> -define(CONNECTOR_IMPL, emqx_bridge_v2_dummy_connector). init_mocks() -> - case emqx_release:edition() of - ee -> - meck:new(emqx_connector_ee_schema, [passthrough, no_link]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), - ok; - ce -> - meck:new(emqx_connector_resource, [passthrough, no_link]), - meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), - ok - end, + meck:new(emqx_connector_resource, [passthrough, no_link]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 9b109b732..f0848b599 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -57,38 +57,7 @@ when ParsedConfig :: #{atom() => any()}, Context :: #{atom() => any()}. -optional_callbacks([connector_config/2]). --if(?EMQX_RELEASE_EDITION == ee). connector_to_resource_type(ConnectorType) -> - try - emqx_connector_ee_schema:resource_type(ConnectorType) - catch - error:{unknown_connector_type, _} -> - %% maybe it's a CE connector - connector_to_resource_type_ce(ConnectorType) - end. - -connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> - connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(ConnectorType) -> - case emqx_connector_ee_schema:connector_impl_module(ConnectorType) of - undefined -> - emqx_connector_info:config_transform_module(ConnectorType); - Module -> - Module - end. --else. - -connector_to_resource_type(ConnectorType) -> - connector_to_resource_type_ce(ConnectorType). - -connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> - connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(ConnectorType) -> - emqx_connector_info:config_transform_module(ConnectorType). - --endif. - -connector_to_resource_type_ce(ConnectorType) -> try emqx_connector_info:resource_callback_module(ConnectorType) catch @@ -96,6 +65,11 @@ connector_to_resource_type_ce(ConnectorType) -> error({unknown_connector_type, ConnectorType}) end. +connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> + connector_impl_module(binary_to_atom(ConnectorType, utf8)); +connector_impl_module(ConnectorType) -> + emqx_connector_info:config_transform_module(ConnectorType). + resource_id(ConnectorId) when is_binary(ConnectorId) -> <<"connector:", ConnectorId/binary>>. diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl deleted file mode 100644 index 8e5f1c001..000000000 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ /dev/null @@ -1,49 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_connector_ee_schema). - --if(?EMQX_RELEASE_EDITION == ee). - --export([ - resource_type/1, - connector_impl_module/1 -]). - --import(hoconsc, [mk/2, enum/1, ref/2]). - --export([ - api_schemas/1, - fields/1, - schema_modules/0, - namespace/0 -]). - -resource_type(Type) when is_binary(Type) -> - resource_type(binary_to_atom(Type, utf8)); -resource_type(Type) -> - error({unknown_connector_type, Type}). - -%% For connectors that need to override connector configurations. -connector_impl_module(ConnectorType) when is_binary(ConnectorType) -> - connector_impl_module(binary_to_atom(ConnectorType, utf8)); -connector_impl_module(_ConnectorType) -> - undefined. - -namespace() -> undefined. - -fields(connectors) -> - connector_structs(). - -connector_structs() -> - []. - -schema_modules() -> - []. - -api_schemas(_Method) -> - []. - --else. - --endif. diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 7bfcab254..c524a8737 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -58,38 +58,6 @@ -export([examples/1]). --if(?EMQX_RELEASE_EDITION == ee). -enterprise_api_schemas(Method) -> - %% We *must* do this to ensure the module is really loaded, especially when we use - %% `call_hocon' from `nodetool' to generate initial configurations. - _ = emqx_connector_ee_schema:module_info(), - case erlang:function_exported(emqx_connector_ee_schema, api_schemas, 1) of - true -> emqx_connector_ee_schema:api_schemas(Method); - false -> [] - end. - -enterprise_fields_connectors() -> - %% We *must* do this to ensure the module is really loaded, especially when we use - %% `call_hocon' from `nodetool' to generate initial configurations. - _ = emqx_connector_ee_schema:module_info(), - case erlang:function_exported(emqx_connector_ee_schema, fields, 1) of - true -> - emqx_connector_ee_schema:fields(connectors); - false -> - [] - end. - --else. - -enterprise_api_schemas(_Method) -> []. - -enterprise_fields_connectors() -> []. - --endif. - -api_schemas(_Method) -> - []. - api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. @@ -105,15 +73,7 @@ examples(Method) -> end, lists:foldl(Fun, #{}, schema_modules()). --if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - emqx_connector_ee_schema:schema_modules() ++ connector_info_schema_modules(). --else. -schema_modules() -> - connector_info_schema_modules(). --endif. - -connector_info_schema_modules() -> ConnectorTypes = emqx_connector_info:connector_types(), [ emqx_connector_info:schema_module(Type) @@ -417,10 +377,8 @@ post_request() -> api_schema("post"). api_schema(Method) -> - CE = api_schemas(Method), - EE = enterprise_api_schemas(Method), InfoModSchemas = emqx_connector_info_api_schemas(Method), - hoconsc:union(connector_api_union(CE ++ EE ++ InfoModSchemas)). + hoconsc:union(connector_api_union(InfoModSchemas)). emqx_connector_info_api_schemas(Method) -> ConnectorTypes = emqx_connector_info:connector_types(), @@ -472,7 +430,7 @@ roots() -> end. fields(connectors) -> - [] ++ enterprise_fields_connectors() ++ connector_info_fields_connectors(); + connector_info_fields_connectors(); fields("node_status") -> [ node_name(), diff --git a/apps/emqx_connector/test/emqx_connector_SUITE.erl b/apps/emqx_connector/test/emqx_connector_SUITE.erl index 5ca5fb315..7df49bd82 100644 --- a/apps/emqx_connector/test/emqx_connector_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_SUITE.erl @@ -44,14 +44,14 @@ end_per_testcase(TestCase, Config) -> %% the 2 test cases below are based on kafka connector which is ee only -if(?EMQX_RELEASE_EDITION == ee). t_connector_lifecycle({init, Config}) -> - meck:new(emqx_connector_ee_schema, [passthrough]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR), + meck:new(emqx_connector_resource, [passthrough]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), meck:expect(?CONNECTOR, on_get_status, 2, connected), - [{mocked_mods, [?CONNECTOR, emqx_connector_ee_schema]} | Config]; + [{mocked_mods, [?CONNECTOR, emqx_connector_resource]} | Config]; t_connector_lifecycle({'end', Config}) -> MockedMods = ?config(mocked_mods, Config), meck:unload(MockedMods), @@ -164,8 +164,8 @@ t_connector_lifecycle(_Config) -> ok. t_remove_fail({'init', Config}) -> - meck:new(emqx_connector_ee_schema, [passthrough]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR), + meck:new(emqx_connector_resource, [passthrough]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), @@ -228,8 +228,8 @@ t_remove_fail(_Config) -> ok. t_create_with_bad_name_direct_path({init, Config}) -> - meck:new(emqx_connector_ee_schema, [passthrough]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR), + meck:new(emqx_connector_resource, [passthrough]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), @@ -259,8 +259,8 @@ t_create_with_bad_name_direct_path(_Config) -> ok. t_create_with_bad_name_root_path({init, Config}) -> - meck:new(emqx_connector_ee_schema, [passthrough]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR), + meck:new(emqx_connector_resource, [passthrough]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), @@ -293,8 +293,8 @@ t_create_with_bad_name_root_path(_Config) -> ok. t_no_buffer_workers({'init', Config}) -> - meck:new(emqx_connector_ee_schema, [passthrough]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR), + meck:new(emqx_connector_resource, [passthrough]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 82fdc4be7..2b1ada37b 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -221,8 +221,8 @@ end_per_testcase(TestCase, Config) -> -define(CONNECTOR_IMPL, dummy_connector_impl). init_mocks(_TestCase) -> - meck:new(emqx_connector_ee_schema, [passthrough, no_link]), - meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), + meck:new(emqx_connector_resource, [passthrough, no_link]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( @@ -265,7 +265,7 @@ init_mocks(_TestCase) -> emqx_bridge_v2:get_channels_for_connector(ResId) end ), - [?CONNECTOR_IMPL, emqx_connector_ee_schema]. + [?CONNECTOR_IMPL, emqx_connector_resource]. clear_resources(_) -> lists:foreach( From 4bfc51c052b147296433dd232121400ec7803821 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 21 Mar 2024 16:08:31 +0100 Subject: [PATCH 21/23] refactor(emqx_connector_info): improve documentation and type specs --- .../src/emqx_bridge_azure_event_hub.erl | 2 ++ .../src/emqx_bridge_cassandra.erl | 2 ++ .../src/emqx_bridge_clickhouse.erl | 2 ++ .../src/emqx_bridge_confluent_producer.erl | 2 ++ .../src/emqx_bridge_dynamo.erl | 2 ++ .../src/emqx_bridge_es_connector.erl | 2 ++ ...emqx_bridge_gcp_pubsub_consumer_schema.erl | 2 ++ ...emqx_bridge_gcp_pubsub_producer_schema.erl | 2 ++ .../src/emqx_bridge_greptimedb.erl | 2 ++ .../src/emqx_bridge_hstreamdb.erl | 2 ++ .../src/emqx_bridge_http_schema.erl | 2 ++ .../src/emqx_bridge_influxdb.erl | 2 ++ .../src/emqx_bridge_iotdb_connector.erl | 2 ++ .../src/emqx_bridge_kafka.erl | 2 ++ .../src/emqx_bridge_kafka_consumer_schema.erl | 2 ++ .../src/emqx_bridge_kinesis.erl | 2 ++ .../src/emqx_bridge_matrix.erl | 2 ++ .../src/emqx_bridge_mongodb.erl | 2 ++ .../src/emqx_bridge_mqtt_connector_schema.erl | 2 ++ .../src/emqx_bridge_mysql.erl | 2 ++ .../src/emqx_bridge_opents_connector.erl | 2 ++ .../src/emqx_bridge_oracle.erl | 2 ++ .../emqx_bridge_pulsar_connector_schema.erl | 2 ++ .../emqx_bridge_rabbitmq_connector_schema.erl | 2 ++ .../src/emqx_bridge_redis_schema.erl | 2 ++ .../src/emqx_bridge_rocketmq.erl | 2 ++ apps/emqx_bridge_s3/src/emqx_bridge_s3.erl | 2 ++ .../src/emqx_bridge_sqlserver.erl | 2 ++ .../src/emqx_bridge_syskeeper_connector.erl | 2 ++ .../src/emqx_bridge_syskeeper_proxy.erl | 2 ++ .../src/emqx_bridge_tdengine_connector.erl | 2 ++ .../src/emqx_bridge_timescale.erl | 2 ++ .../src/emqx_connector_examples.erl | 22 +++++++++++++++++ .../src/emqx_connector_info.erl | 24 ++++++++++++++++--- .../emqx_postgresql_connector_schema.erl | 2 ++ 35 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_connector/src/emqx_connector_examples.erl diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index 431583b81..213c2331c 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_azure_event_hub). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index cf001217e..d34cb1950 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_cassandra). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl index 1de7bfab1..833c2570d 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_clickhouse). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl index d830ac637..bf01626e3 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_confluent_producer). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl index 01c29a3b3..13828c0f7 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_dynamo). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 256645c31..fc3aa6d3b 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_es_connector). +-behaviour(emqx_connector_examples). + -behaviour(emqx_resource). -include("emqx_bridge_es.hrl"). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_schema.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_schema.erl index 00c527e65..9e3a99add 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_schema.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_schema.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_gcp_pubsub_consumer_schema). +-behaviour(emqx_connector_examples). + -import(hoconsc, [mk/2, ref/2]). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl index f2db2ee49..c42c1023e 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_gcp_pubsub_producer_schema). +-behaviour(emqx_connector_examples). + -import(hoconsc, [mk/2, ref/2]). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index 7dab38620..bfb740cd4 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_greptimedb). +-behaviour(emqx_connector_examples). + -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl index d66077171..7cd7c8240 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_hstreamdb). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include("emqx_bridge_hstreamdb.hrl"). diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index 290e926da..43f3d1748 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -15,6 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_http_schema). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index 33202e458..a62effe51 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_influxdb). +-behaviour(emqx_connector_examples). + -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 58fd66fe1..07c4b0251 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_iotdb_connector). +-behaviour(emqx_connector_examples). + -behaviour(emqx_resource). -include("emqx_bridge_iotdb.hrl"). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 84bbe01c1..27399fd1f 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_kafka). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_schema.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_schema.erl index 266d3f7d9..bc307d28e 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_schema.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_schema.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_kafka_consumer_schema). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl index 06cf76cf5..c55619a7e 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_kinesis). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl index f59269adf..dae2851c9 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_matrix). +-behaviour(emqx_connector_examples). + -include_lib("hocon/include/hoconsc.hrl"). -export([ diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index 0d1bd2981..c81df1334 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_mongodb). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 7233e9e6c..7103e53ee 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -16,6 +16,8 @@ -module(emqx_bridge_mqtt_connector_schema). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index e80916f10..ee7487760 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_mysql). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index af4cba951..c2cb169e4 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_opents_connector). +-behaviour(emqx_connector_examples). + -behaviour(emqx_resource). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl index cd0943dd5..fb485c16b 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_oracle). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_schema.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_schema.erl index f8b7e3909..3cca35025 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_schema.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector_schema.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_pulsar_connector_schema). +-behaviour(emqx_connector_examples). + -export([namespace/0, roots/0, fields/1, desc/1]). -export([connector_examples/1, connector_example_values/0]). diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_schema.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_schema.erl index 02b5ae61c..118c52af4 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_schema.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector_schema.erl @@ -16,6 +16,8 @@ -module(emqx_bridge_rabbitmq_connector_schema). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index 64c166659..32910fb10 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_redis_schema). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -define(TYPE, redis). diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl index fe98a197b..589719486 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_rocketmq). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl index 1864b72eb..5d7e176e3 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_s3). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include("emqx_bridge_s3.hrl"). diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.erl index 850f97487..e9df1fdb6 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_sqlserver). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 2e8f9981a..045af348f 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_syskeeper_connector). +-behaviour(emqx_connector_examples). + -behaviour(emqx_resource). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl index 80f338c69..9275d5e16 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_syskeeper_proxy). +-behaviour(emqx_connector_examples). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 7d2815f54..3988ccd79 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -4,6 +4,8 @@ -module(emqx_bridge_tdengine_connector). +-behaviour(emqx_connector_examples). + -behaviour(emqx_resource). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl index ed51c737e..2f5a29a30 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_timescale). +-behaviour(emqx_connector_examples). + -include_lib("hocon/include/hoconsc.hrl"). -export([ diff --git a/apps/emqx_connector/src/emqx_connector_examples.erl b/apps/emqx_connector/src/emqx_connector_examples.erl new file mode 100644 index 000000000..906204c69 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_examples.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_examples). + +%% Should return examples for the connector HTTP API. One can use the helper +%% function emqx_connector_schema:connector_values/3 to generate these +%% examples. See emqx_bridge_oracle:connector_examples/1 for an example. +-callback connector_examples(Method :: get | post | put) -> [map()]. diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index c53fede62..25039dee7 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -31,15 +31,33 @@ -export([clean_cache/0]). +%% The type name for the conncector -callback type_name() -> atom(). + +%% The action, source and bridge_v1 types that are using the connector -callback bridge_types() -> [atom()]. + +%% The implementation of the emqx_resource behavior for the connector -callback resource_callback_module() -> atom(). + +%% The module containing implementation of the emqx_connector_examples behavior. -callback schema_module() -> atom(). + +%% The hocon configuration schema for the connector. This schema should usually +%% contain a reference to the schema_module(). See +%% emqx_bridge_oracle_connector_info:config_schema/0 for an example. -callback config_schema() -> term(). --callback api_schema([char()]) -> term(). + +%% Should return the hocon configuration schema for the connector HTTP method. +%% Note that one can use the helper functions emqx_connector_schema:api_ref/3 +%% and emqx_connector_schema:api_fields/3 to implement this. See +%% emqx_bridge_oracle_connector_info:api_schema/1 and the corresponding clause +%% in emqx_bridge_oracle:fields/1 for an example. Method is "get", "post" or "put". +-callback api_schema(Method :: string()) -> term(). + %% Optional callback that should return a module with an exported %% connector_config/2 function. If present this function will be used to -%% transfrom the connector configuration. See the callback connector_config/2 +%% transform the connector configuration. See the callback connector_config/2 %% in emqx_connector_resource for more information. -callback config_transform_module() -> atom(). @@ -48,7 +66,7 @@ ]). %% ==================================================================== -%% HardCoded list of info modules for connectors +%% Hard coded list of info modules for connectors %% TODO: Remove this list once we have made sure that all relevants %% apps are loaded before this module is called. %% ==================================================================== diff --git a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl index 4653c269e..fe5c4cd78 100644 --- a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl +++ b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl @@ -16,6 +16,8 @@ -module(emqx_postgresql_connector_schema). +-behaviour(emqx_connector_examples). + -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_postgresql/include/emqx_postgresql.hrl"). From 2b684c7bcc4e8afff3439393dcac857aabd75e76 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 22 Mar 2024 09:29:26 +0100 Subject: [PATCH 22/23] fix(emqx_connector_info): refactoring based on thalesmg's comments --- apps/emqx_bridge/src/emqx_action_info.erl | 2 +- .../src/emqx_bridge_azure_event_hub_action_info.erl | 6 ++++-- apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl | 2 +- apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src | 4 ++-- ..._info.erl => emqx_bridge_kafka_producer_action_info.erl} | 2 +- ...fo.erl => emqx_bridge_kafka_producer_connector_info.erl} | 2 +- apps/emqx_connector/src/emqx_connector_info.erl | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) rename apps/emqx_bridge_kafka/src/{emqx_bridge_kafka_action_info.erl => emqx_bridge_kafka_producer_action_info.erl} (97%) rename apps/emqx_bridge_kafka/src/{emqx_bridge_kafka_connector_info.erl => emqx_bridge_kafka_producer_connector_info.erl} (95%) diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index c6282fc18..464b2e429 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -91,7 +91,7 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_dynamo_action_info, emqx_bridge_gcp_pubsub_consumer_action_info, emqx_bridge_gcp_pubsub_producer_action_info, - emqx_bridge_kafka_action_info, + emqx_bridge_kafka_producer_action_info, emqx_bridge_kafka_consumer_action_info, emqx_bridge_kinesis_action_info, emqx_bridge_hstreamdb_action_info, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_action_info.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_action_info.erl index 4361a3bee..6c472b479 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_action_info.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub_action_info.erl @@ -24,9 +24,11 @@ connector_type_name() -> azure_event_hub_producer. schema_module() -> emqx_bridge_azure_event_hub. connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> - emqx_bridge_kafka_action_info:connector_action_config_to_bridge_v1_config( + emqx_bridge_kafka_producer_action_info:connector_action_config_to_bridge_v1_config( ConnectorConfig, ActionConfig ). bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) -> - emqx_bridge_kafka_action_info:bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName). + emqx_bridge_kafka_producer_action_info:bridge_v1_config_to_action_config( + BridgeV1Conf, ConnectorName + ). diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl index 4542c9382..4fe507047 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl @@ -35,7 +35,7 @@ config_schema() -> )}. schema_module() -> - emqx_bridge_azure_event_hub. + emqx_bridge_es_connector. api_schema(Method) -> emqx_connector_schema:api_ref( diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index 90ac23e81..34b8dfa06 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -14,12 +14,12 @@ ]}, {env, [ {emqx_action_info_modules, [ - emqx_bridge_kafka_action_info, + emqx_bridge_kafka_producer_action_info, emqx_bridge_kafka_consumer_action_info ]}, {emqx_connector_info_modules, [ emqx_bridge_kafka_consumer_connector_info, - emqx_bridge_kafka_connector_info + emqx_bridge_kafka_producer_connector_info ]} ]}, {modules, []}, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_action_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl similarity index 97% rename from apps/emqx_bridge_kafka/src/emqx_bridge_kafka_action_info.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl index 2c7810028..37a061072 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_action_info.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_kafka_action_info). +-module(emqx_bridge_kafka_producer_action_info). -behaviour(emqx_action_info). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_connector_info.erl similarity index 95% rename from apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_connector_info.erl index d0b013cc3..e320a42c6 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_connector_info.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_connector_info.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_kafka_connector_info). +-module(emqx_bridge_kafka_producer_connector_info). -behaviour(emqx_connector_info). diff --git a/apps/emqx_connector/src/emqx_connector_info.erl b/apps/emqx_connector/src/emqx_connector_info.erl index 25039dee7..766f34168 100644 --- a/apps/emqx_connector/src/emqx_connector_info.erl +++ b/apps/emqx_connector/src/emqx_connector_info.erl @@ -81,7 +81,7 @@ hard_coded_connector_info_modules_ee() -> emqx_bridge_gcp_pubsub_producer_connector_info, emqx_bridge_hstreamdb_connector_info, emqx_bridge_kafka_consumer_connector_info, - emqx_bridge_kafka_connector_info, + emqx_bridge_kafka_producer_connector_info, emqx_bridge_kinesis_connector_info, emqx_bridge_matrix_connector_info, emqx_bridge_pgsql_connector_info, From e39fa5eaa9d4a7c0beddbbe5318e891cb6c9f35e Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 25 Mar 2024 10:44:33 +0100 Subject: [PATCH 23/23] docs(es_connector_info): add copyright header --- apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl index 4fe507047..2ca2a73ad 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector_info.erl @@ -1,3 +1,7 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + -module(emqx_bridge_es_connector_info). -behaviour(emqx_connector_info).