From 6143b3a2be6acc99ee3fd37834f19506cd5889fb Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:08:45 +0100 Subject: [PATCH 1/5] fix(emqx_bridge_mongodb): use correct config when calling downgrade_type Also limit result in mongodb ..._to_bridge_v1_config to not include more fields than what's in the schema (most notably `description`). --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_lib.erl | 3 ++- .../src/emqx_bridge_mongodb_action_info.erl | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 2343e3ccf..9ec0d440c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -913,7 +913,7 @@ format_resource( redact( maps:merge( RawConfFull#{ - type => downgrade_type(Type, RawConf), + type => downgrade_type(Type, emqx_bridge_lib:get_conf(Type, BridgeName)), name => maps:get(<<"name">>, RawConf, BridgeName), node => Node }, diff --git a/apps/emqx_bridge/src/emqx_bridge_lib.erl b/apps/emqx_bridge/src/emqx_bridge_lib.erl index ed8e918fa..9386a38d3 100644 --- a/apps/emqx_bridge/src/emqx_bridge_lib.erl +++ b/apps/emqx_bridge/src/emqx_bridge_lib.erl @@ -18,7 +18,8 @@ -export([ maybe_withdraw_rule_action/3, upgrade_type/1, - downgrade_type/2 + downgrade_type/2, + get_conf/2 ]). %% @doc A bridge can be used as a rule action. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl index 9fa19add2..be60959e8 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl @@ -26,19 +26,21 @@ -define(SCHEMA_MODULE, emqx_bridge_mongodb). -connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> - fix_v1_type( - maps:merge( +connector_action_config_to_bridge_v1_config( + #{<<"parameters">> := #{<<"mongo_type">> := MongoType}} = ConnectorConfig, + ActionConfig +) -> + MergedConfig = + emqx_utils_maps:deep_merge( maps:without( [<<"connector">>], emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) ), emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) - ) - ). - -fix_v1_type(#{<<"mongo_type">> := MongoType} = Conf) -> - Conf#{<<"type">> => v1_type(MongoType)}. + ), + BridgeV1Type = v1_type(MongoType), + BridgeV1Keys = schema_keys(BridgeV1Type), + maps:with(BridgeV1Keys, MergedConfig). bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(mongodb_action), From be31486983ce2c292a1087c4a8168fc9c9a08f56 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:40:24 +0100 Subject: [PATCH 2/5] fix(emqx_utils): use deep_merge in unindent --- apps/emqx_utils/src/emqx_utils_maps.erl | 2 +- apps/emqx_utils/test/emqx_utils_maps_tests.erl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index 1d97a926a..47b21a90a 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -345,7 +345,7 @@ indent(IndentKey, PickKeys, Map) -> -spec unindent(term(), map()) -> map(). unindent(Key, Map) -> - maps:merge( + deep_merge( maps:remove(Key, Map), maps:get(Key, Map, #{}) ). diff --git a/apps/emqx_utils/test/emqx_utils_maps_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl index 2778b5257..8b6e29235 100644 --- a/apps/emqx_utils/test/emqx_utils_maps_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -170,5 +170,9 @@ map_indent_unindent_test_() -> ?_assertEqual( #{a => 1, b => 2}, unindent(c, M) + ), + ?_assertEqual( + #{a => #{c => 3, d => 4}}, + unindent(b, #{a => #{c => 3}, b => #{a => #{d => 4}}}) ) ]. From 31a7301f6f133e61fcb2fe5163436317bbf4d706 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:50:38 +0100 Subject: [PATCH 3/5] refactor(emqx_bridge): be more generic in to_bridge_v1_config --- apps/emqx_bridge/src/emqx_action_info.erl | 19 +++++++- apps/emqx_bridge/src/emqx_bridge_v2.erl | 45 +++++++------------ .../src/emqx_bridge_mongodb_action_info.erl | 17 ------- 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 9376eeef3..7d82f76d2 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -26,6 +26,7 @@ bridge_v1_type_to_action_type/1, is_action_type/1, registered_schema_modules/0, + connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, has_custom_connector_action_config_to_bridge_v1_config/1, bridge_v1_config_to_connector_config/2, @@ -168,8 +169,22 @@ has_custom_connector_action_config_to_bridge_v1_config(ActionOrBridgeType) -> connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), - %% should only be called if defined - Module:connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig). + case erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2) of + true -> + Module:connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig); + false -> + connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) + end. + +connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> + Merged = emqx_utils_maps:deep_merge( + maps:without( + [<<"connector">>], + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) + ), + emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) + ), + maps:without([<<"description">>], Merged). has_custom_bridge_v1_config_to_connector_config(ActionOrBridgeType) -> Module = get_action_info_module(ActionOrBridgeType), diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index a9aeb0f8e..00d3ef6ec 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1112,40 +1112,25 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. +connector_raw_config(Connector, ConnectorType) -> + get_raw_with_defaults(Connector, ConnectorType, <<"connectors">>, emqx_connector_schema). + +action_raw_config(Action, ActionType) -> + get_raw_with_defaults(Action, ActionType, <<"actions">>, emqx_bridge_v2_schema). + +get_raw_with_defaults(Config, Type, TopLevelConf, SchemaModule) -> + RawConfig = maps:get(raw_config, Config), + fill_defaults(Type, RawConfig, TopLevelConf, SchemaModule). + bridge_v1_lookup_and_transform_helper( BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> - ConnectorRawConfig1 = maps:get(raw_config, Connector), - ConnectorRawConfig2 = fill_defaults( - ConnectorType, - ConnectorRawConfig1, - <<"connectors">>, - emqx_connector_schema + ConnectorRawConfig = connector_raw_config(Connector, ConnectorType), + ActionRawConfig = action_raw_config(Action, ActionType), + BridgeV1Config = emqx_action_info:connector_action_config_to_bridge_v1_config( + BridgeV1Type, ConnectorRawConfig, ActionRawConfig ), - ActionRawConfig1 = maps:get(raw_config, Action), - ActionRawConfig2 = fill_defaults( - ActionType, - ActionRawConfig1, - <<"actions">>, - emqx_bridge_v2_schema - ), - BridgeV1ConfigFinal = - case - emqx_action_info:has_custom_connector_action_config_to_bridge_v1_config(BridgeV1Type) - of - false -> - BridgeV1Config1 = maps:remove(<<"connector">>, ActionRawConfig2), - %% Move parameters to the top level - ParametersMap = maps:get(<<"parameters">>, BridgeV1Config1, #{}), - BridgeV1Config2 = maps:remove(<<"parameters">>, BridgeV1Config1), - BridgeV1Config3 = emqx_utils_maps:deep_merge(BridgeV1Config2, ParametersMap), - emqx_utils_maps:deep_merge(ConnectorRawConfig2, BridgeV1Config3); - true -> - emqx_action_info:connector_action_config_to_bridge_v1_config( - BridgeV1Type, ConnectorRawConfig2, ActionRawConfig2 - ) - end, - BridgeV1Tmp = maps:put(raw_config, BridgeV1ConfigFinal, Action), + BridgeV1Tmp = maps:put(raw_config, BridgeV1Config, Action), BridgeV1 = maps:remove(status, BridgeV1Tmp), BridgeV2Status = maps:get(status, Action, undefined), BridgeV2Error = maps:get(error, Action, undefined), diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl index be60959e8..060e6a17a 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_action_info.erl @@ -10,7 +10,6 @@ -export([ bridge_v1_config_to_action_config/2, bridge_v1_config_to_connector_config/1, - connector_action_config_to_bridge_v1_config/2, action_type_name/0, bridge_v1_type_name/0, connector_type_name/0, @@ -26,22 +25,6 @@ -define(SCHEMA_MODULE, emqx_bridge_mongodb). -connector_action_config_to_bridge_v1_config( - #{<<"parameters">> := #{<<"mongo_type">> := MongoType}} = ConnectorConfig, - ActionConfig -) -> - MergedConfig = - emqx_utils_maps:deep_merge( - maps:without( - [<<"connector">>], - emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) - ), - emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) - ), - BridgeV1Type = v1_type(MongoType), - BridgeV1Keys = schema_keys(BridgeV1Type), - maps:with(BridgeV1Keys, MergedConfig). - bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(mongodb_action), ActionParametersKeys = schema_keys(action_parameters), From 8e9cb6a928e97c457c7ee1339da8dea11869f11d Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:54:10 +0100 Subject: [PATCH 4/5] style(emqx_bridge_mysql): minor cleanup --- apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl | 2 +- apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index 05c782b96..6f1036600 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -129,7 +129,7 @@ fields(action) -> {mysql, mk( hoconsc:map(name, ref(?MODULE, mysql_action)), - #{desc => <<"MySQL Action Config">>, required => false} + #{desc => ?DESC("mysql_action"), required => false} )}; fields(mysql_action) -> emqx_bridge_v2_schema:make_producer_action_schema( diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index f1b3f8260..65413817d 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -773,7 +773,6 @@ t_missing_table(Config) -> ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - %Timeout = 1000, ResourceID = emqx_bridge_v2:id(BridgeType, Name), Request = {ResourceID, SentData}, Result = From a7c344ba292eb77323a459e7a4ec7071e86d26cf Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 11 Dec 2023 14:58:09 +0100 Subject: [PATCH 5/5] fix(emqx_bridge): remove unused --- apps/emqx_bridge/src/emqx_action_info.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 7d82f76d2..c6304d9f7 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -28,7 +28,6 @@ registered_schema_modules/0, connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, - has_custom_connector_action_config_to_bridge_v1_config/1, bridge_v1_config_to_connector_config/2, has_custom_bridge_v1_config_to_connector_config/1, bridge_v1_config_to_action_config/3, @@ -163,10 +162,6 @@ registered_schema_modules() -> Schemas = maps:get(action_type_to_schema_module, InfoMap), maps:to_list(Schemas). -has_custom_connector_action_config_to_bridge_v1_config(ActionOrBridgeType) -> - Module = get_action_info_module(ActionOrBridgeType), - erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2). - connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), case erlang:function_exported(Module, connector_action_config_to_bridge_v1_config, 2) of