From 38ef99caf59f3584e63c4feac242e81da15622c2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 11:49:59 -0300 Subject: [PATCH 1/4] chore(dashboard): bump dashboard -> e1.0.7-beta.3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ad4421aa..785458601 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ endif # Dashbord version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.2.4-1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7-beta.3 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used # In make 4.4+, for backward-compatibility the value from the original environment is used. From dcccc0910aad87d289dff5bd2a2aa6c7625a2b36 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 11:44:19 -0300 Subject: [PATCH 2/4] fix(pulsar): mark whole auth struct as sensitive (r5.0) Fixes https://emqx.atlassian.net/browse/EMQX-9900 I tried to patch hocon itself to filter the sensitive data, but the way it's currently structured doesn't seem to keep that field metadata. So, for now, we can just mark the whole auth union as sensitive. --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 5a87d8a0c..56c20130c 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -46,7 +46,14 @@ fields(config) -> )}, {authentication, mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ - default => none, desc => ?DESC("authentication") + default => none, + %% must mark this whole union as sensitive because + %% hocon ignores the `sensitive' metadata in struct + %% fields... Also, when trying to type check a struct + %% that doesn't match the intended type, it won't have + %% sensitivity information from sibling types. + sensitive => true, + desc => ?DESC("authentication") })} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(producer_opts) -> From 659cf64ad7f40e9f436b7dd234a5c55b124ea098 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 17:56:53 -0300 Subject: [PATCH 3/4] feat(pulsar): use an union member selector for better error messages --- .../src/emqx_bridge_pulsar.erl | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 56c20130c..7d1b20d24 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -45,16 +45,19 @@ fields(config) -> } )}, {authentication, - mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ - default => none, - %% must mark this whole union as sensitive because - %% hocon ignores the `sensitive' metadata in struct - %% fields... Also, when trying to type check a struct - %% that doesn't match the intended type, it won't have - %% sensitivity information from sibling types. - sensitive => true, - desc => ?DESC("authentication") - })} + mk( + hoconsc:union(fun auth_union_member_selector/1), + #{ + default => none, + %% must mark this whole union as sensitive because + %% hocon ignores the `sensitive' metadata in struct + %% fields... Also, when trying to type check a struct + %% that doesn't match the intended type, it won't have + %% sensitivity information from sibling types. + sensitive => true, + desc => ?DESC("authentication") + } + )} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(producer_opts) -> [ @@ -233,3 +236,21 @@ override_default(OriginalFn, NewDefault) -> (default) -> NewDefault; (Field) -> OriginalFn(Field) end. + +auth_union_member_selector(all_union_members) -> + [none, ref(auth_basic), ref(auth_token)]; +auth_union_member_selector({value, V}) -> + case V of + #{<<"password">> := _} -> + [ref(auth_basic)]; + #{<<"jwt">> := _} -> + [ref(auth_token)]; + <<"none">> -> + [none]; + _ -> + Expected = "none | basic | token", + throw(#{ + field_name => authentication, + expected => Expected + }) + end. From 5d5c16a56d7e62a356b10cf855d09a4ab26a2822 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 17:57:29 -0300 Subject: [PATCH 4/4] feat(bridges): use union member selector function for better error messages --- .../src/schema/emqx_bridge_schema.erl | 34 ++++++++++- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 61 +++++++++++-------- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index b590f0cd4..90e62764a 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -51,11 +51,39 @@ post_request() -> api_schema(Method) -> Broker = [ - ref(Mod, Method) - || Mod <- [emqx_bridge_webhook_schema, emqx_bridge_mqtt_schema] + {Type, ref(Mod, Method)} + || {Type, Mod} <- [ + {<<"webhook">>, emqx_bridge_webhook_schema}, + {<<"mqtt">>, emqx_bridge_mqtt_schema} + ] ], EE = ee_api_schemas(Method), - hoconsc:union(Broker ++ EE). + hoconsc:union(bridge_api_union(Broker ++ EE)). + +bridge_api_union(Refs) -> + Index = maps:from_list(Refs), + fun + (all_union_members) -> + maps:values(Index); + ({value, V}) -> + case V of + #{<<"type">> := T} -> + case maps:get(T, Index, undefined) of + undefined -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }); + Ref -> + [Ref] + end; + _ -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }) + end + end. -if(?EMQX_RELEASE_EDITION == ee). ee_api_schemas(Method) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 6e2dbcbce..bff42dd98 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 17ffe9b9b..804e7d814 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,33 +14,37 @@ api_schemas(Method) -> [ - ref(emqx_bridge_gcp_pubsub, Method), - ref(emqx_bridge_kafka, Method ++ "_consumer"), - ref(emqx_bridge_kafka, Method ++ "_producer"), - ref(emqx_bridge_cassandra, Method), - ref(emqx_ee_bridge_mysql, Method), - ref(emqx_bridge_pgsql, Method), - ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), - ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), - ref(emqx_ee_bridge_mongodb, Method ++ "_single"), - ref(emqx_ee_bridge_hstreamdb, Method), - ref(emqx_bridge_influxdb, Method ++ "_api_v1"), - ref(emqx_bridge_influxdb, Method ++ "_api_v2"), - ref(emqx_ee_bridge_redis, Method ++ "_single"), - ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), - ref(emqx_ee_bridge_redis, Method ++ "_cluster"), - ref(emqx_bridge_timescale, Method), - ref(emqx_bridge_matrix, Method), - ref(emqx_bridge_tdengine, Method), - ref(emqx_ee_bridge_clickhouse, Method), - ref(emqx_bridge_dynamo, Method), - ref(emqx_bridge_rocketmq, Method), - ref(emqx_bridge_sqlserver, Method), - ref(emqx_bridge_opents, Method), - ref(emqx_bridge_pulsar, Method ++ "_producer"), - ref(emqx_bridge_oracle, Method), - ref(emqx_bridge_iotdb, Method), - ref(emqx_bridge_rabbitmq, Method) + %% We need to map the `type' field of a request (binary) to a + %% bridge schema module. + api_ref(emqx_bridge_gcp_pubsub, <<"gcp_pubsub">>, Method), + api_ref(emqx_bridge_kafka, <<"kafka_consumer">>, Method ++ "_consumer"), + %% TODO: rename this to `kafka_producer' after alias support is added + %% to hocon; keeping this as just `kafka' for backwards compatibility. + api_ref(emqx_bridge_kafka, <<"kafka">>, Method ++ "_producer"), + api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method), + api_ref(emqx_ee_bridge_mysql, <<"mysql">>, Method), + api_ref(emqx_bridge_pgsql, <<"pgsql">>, Method), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), + api_ref(emqx_ee_bridge_redis, <<"redis_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"), + api_ref(emqx_ee_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"), + api_ref(emqx_bridge_timescale, <<"timescale">>, Method), + api_ref(emqx_bridge_matrix, <<"matrix">>, Method), + api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method), + api_ref(emqx_ee_bridge_clickhouse, <<"clickhouse">>, Method), + api_ref(emqx_bridge_dynamo, <<"dynamo">>, Method), + api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method), + api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method), + api_ref(emqx_bridge_opents, <<"opents">>, Method), + api_ref(emqx_bridge_pulsar, <<"pulsar_producer">>, Method ++ "_producer"), + api_ref(emqx_bridge_oracle, <<"oracle">>, Method), + api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), + api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method) ]. schema_modules() -> @@ -338,3 +342,6 @@ rabbitmq_structs() -> } )} ]. + +api_ref(Module, Type, Method) -> + {Type, ref(Module, Method)}.