From bdce32e713f23234ecf650b80eb3ecaae46c585b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 13:05:25 +0800 Subject: [PATCH] refactor(cassandra): move cassandra bridge into its own app --- apps/emqx_bridge_cassandra/docker-ct | 2 + apps/emqx_bridge_cassandra/rebar.config | 11 +++++ .../src/emqx_bridge_cassandra.app.src | 2 +- .../src/emqx_bridge_cassandra.erl | 4 +- .../src/emqx_bridge_cassandra_impl.erl | 2 +- .../test/emqx_bridge_cassandra_SUITE.erl | 6 +-- .../test/emqx_bridge_cassandra_impl_SUITE.erl | 4 +- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/rebar.config | 3 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++-- lib-ee/emqx_ee_connector/docker-ct | 1 - .../src/emqx_ee_connector.app.src | 1 - mix.exs | 13 ++++++ rebar.config.erl | 26 ++++++++++++ ...assa.hocon => emqx_bridge_cassandra.hocon} | 2 +- ...hocon => emqx_bridge_cassandra_impl.hocon} | 2 +- scripts/ct/run.sh | 42 +++++++++++++++++++ 17 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_bridge_cassandra/docker-ct create mode 100644 apps/emqx_bridge_cassandra/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl => apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl (97%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl => apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl => apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl (99%) rename lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl => apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl (98%) rename rel/i18n/{emqx_ee_bridge_cassa.hocon => emqx_bridge_cassandra.hocon} (97%) rename rel/i18n/{emqx_ee_connector_cassa.hocon => emqx_bridge_cassandra_impl.hocon} (92%) diff --git a/apps/emqx_bridge_cassandra/docker-ct b/apps/emqx_bridge_cassandra/docker-ct new file mode 100644 index 000000000..2626b4068 --- /dev/null +++ b/apps/emqx_bridge_cassandra/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +cassandra diff --git a/apps/emqx_bridge_cassandra/rebar.config b/apps/emqx_bridge_cassandra/rebar.config new file mode 100644 index 000000000..b8bfc7dd6 --- /dev/null +++ b/apps/emqx_bridge_cassandra/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_cassandra]} +]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 1ed65ea9f..58e4a1984 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise Cassandra Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, ecql]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 26c6de04d..32dc11839 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa). +-module(emqx_bridge_cassandra). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -88,7 +88,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_ee_connector_cassa:fields(config) -- + (emqx_bridge_cassandra_impl:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", cassandra); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl index 397532f47..8fe329a98 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa). +-module(emqx_bridge_cassandra_impl). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 4711d1981..482f2c1f2 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa_SUITE). +-module(emqx_bridge_cassandra_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -57,7 +57,7 @@ %% CASSA_TCP_HOST=127.0.0.1 CASSA_TCP_PORT=19042 \ %% CASSA_TLS_HOST=127.0.0.1 CASSA_TLS_PORT=19142 \ %% PROXY_HOST=127.0.0.1 ./rebar3 as test ct -c -v --name ct@127.0.0.1 \ -%% --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +%% --suite apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl %% %%------------------------------------------------------------------------------ @@ -590,7 +590,7 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - %% emqx_ee_connector_cassa will send missed data as a `null` atom + %% emqx_bridge_cassandra_impl will send missed data as a `null` atom %% to ecql driver ?check_trace( begin diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl index f2647d756..faaee3c63 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa_SUITE). +-module(emqx_bridge_cassandra_impl_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -16,7 +16,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). +-define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_impl). %% This test SUITE requires a running cassandra instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 35d6b9d5b..aa19a495f 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -10,5 +10,4 @@ tdengine clickhouse dynamo rocketmq -cassandra sqlserver diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index b26df658a..358ff3bc8 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} - , {emqx_connector, {path, "../../apps/emqx_connector"}} +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} , {emqx_utils, {path, "../emqx_utils"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 9465464d9..e8be79cdc 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -17,6 +17,7 @@ api_schemas(Method) -> ref(emqx_bridge_gcp_pubsub, Method), ref(emqx_bridge_kafka, Method ++ "_consumer"), ref(emqx_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_cassandra, Method), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -34,13 +35,13 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_cassa, Method), ref(emqx_ee_bridge_sqlserver, Method) ]. schema_modules() -> [ emqx_bridge_kafka, + emqx_bridge_cassandra, emqx_ee_bridge_hstreamdb, emqx_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, @@ -54,7 +55,6 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_cassa, emqx_ee_bridge_sqlserver ]. @@ -75,6 +75,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; +resource_type(cassandra) -> emqx_bridge_cassandra_impl; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -93,7 +94,6 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(cassandra) -> emqx_ee_connector_cassa; resource_type(sqlserver) -> emqx_ee_connector_sqlserver. fields(bridges) -> @@ -148,7 +148,7 @@ fields(bridges) -> )}, {cassandra, mk( - hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), + hoconsc:map(name, ref(emqx_bridge_cassandra, "config")), #{ desc => <<"Cassandra Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index fc8e75e68..cb2f6f028 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,5 +1,4 @@ toxiproxy influxdb clickhouse -cassandra sqlserver diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index ced7ae86a..82f556bdb 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -12,7 +12,6 @@ clickhouse, erlcloud, rocketmq, - ecql, odbc ]}, {env, []}, diff --git a/mix.exs b/mix.exs index f3149a584..c32fa798d 100644 --- a/mix.exs +++ b/mix.exs @@ -156,6 +156,19 @@ defmodule EMQXUmbrella.MixProject do MapSet.new([ :emqx_bridge_kafka, :emqx_bridge_gcp_pubsub + :emqx_bridge_cassandra, + :emqx_bridge_clickhouse, + :emqx_bridge_dynamo, + :emqx_bridge_hstreamdb, + :emqx_bridge_influxdb, + :emqx_bridge_matrix, + :emqx_bridge_mongodb, + :emqx_bridge_mysql, + :emqx_bridge_pgsql, + :emqx_bridge_redis, + :emqx_bridge_rocketmq, + :emqx_bridge_tdengine, + :emqx_bridge_timescale ]) end diff --git a/rebar.config.erl b/rebar.config.erl index 9bdbfb848..88471c39d 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -80,6 +80,19 @@ is_enterprise(ee) -> true. is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; +is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; +is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; +is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; +is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_matrix") -> false; +is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false; +is_community_umbrella_app("apps/emqx_bridge_mysql") -> false; +is_community_umbrella_app("apps/emqx_bridge_pgsql") -> false; +is_community_umbrella_app("apps/emqx_bridge_redis") -> false; +is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false; +is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false; +is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -441,6 +454,19 @@ relx_apps_per_edition(ee) -> emqx_ee_bridge, emqx_bridge_kafka, emqx_bridge_gcp_pubsub, + emqx_bridge_cassandra, + emqx_bridge_clickhouse, + emqx_bridge_dynamo, + emqx_bridge_hstreamdb, + emqx_bridge_influxdb, + emqx_bridge_matrix, + emqx_bridge_mongodb, + emqx_bridge_mysql, + emqx_bridge_pgsql, + emqx_bridge_redis, + emqx_bridge_rocketmq, + emqx_bridge_tdengine, + emqx_bridge_timescale, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_ee_bridge_cassa.hocon b/rel/i18n/emqx_bridge_cassandra.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra.hocon index d86c95a5f..d598d3921 100644 --- a/rel/i18n/emqx_ee_bridge_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_cassa { +emqx_bridge_cassandra { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_cassa.hocon b/rel/i18n/emqx_bridge_cassandra_impl.hocon similarity index 92% rename from rel/i18n/emqx_ee_connector_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra_impl.hocon index bd5fb544c..91157fc09 100644 --- a/rel/i18n/emqx_ee_connector_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra_impl.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_cassa { +emqx_bridge_cassa_impl { keyspace.desc: """Keyspace name to connect to.""" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4e79476e0..aca83ae86 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -112,6 +112,48 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing ee applications export PROFILE='emqx-enterprise' ;; + apps/emqx_bridge_cassandra) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_clickhouse) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_dynamo) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_gcp_pubsub) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_hstreamdb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_influxdb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_matrix) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_mongodb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_mysql) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_pgsql) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_redis) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_rocketmq) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_tdengine) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_timescale) + export PROFILE='emqx-enterprise' + ;; *) export PROFILE="${PROFILE:-emqx}" ;;