diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md index ff870e87d..e74afab3d 100644 --- a/apps/emqx_bridge_clickhouse/README.md +++ b/apps/emqx_bridge_clickhouse/README.md @@ -23,7 +23,7 @@ User can create a rule and easily ingest IoT data into ClickHouse by leveraging - Several APIs are provided for bridge management, which includes create bridge, update bridge, get bridge, stop or restart bridge and list bridges etc. - Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) +- Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. diff --git a/apps/emqx_bridge_clickhouse/docker-ct b/apps/emqx_bridge_clickhouse/docker-ct new file mode 100644 index 000000000..938ba95c6 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/docker-ct @@ -0,0 +1 @@ +clickhouse diff --git a/apps/emqx_bridge_clickhouse/rebar.config b/apps/emqx_bridge_clickhouse/rebar.config new file mode 100644 index 000000000..a8da74b43 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_clickhouse]} +]}. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index a0b409d5b..751628fa4 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.2.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, clickhouse]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl index 56671c586..d317ecc5c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse). +-module(emqx_bridge_clickhouse). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -101,7 +101,7 @@ fields("config") -> } )} ] ++ - emqx_ee_connector_clickhouse:fields(config); + emqx_bridge_clickhouse_connector:fields(config); fields("creation_opts") -> emqx_resource_schema:fields("creation_opts"); fields("post") -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl rename to apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index a7afcd6d5..a2de1d3c9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse). +-module(emqx_bridge_clickhouse_connector). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index 6d4762882..b39296044 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -2,17 +2,16 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_SUITE). -compile(nowarn_export_all). -compile(export_all). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). -include_lib("emqx_connector/include/emqx_connector.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl for how to +%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture %%------------------------------------------------------------------------------ diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl similarity index 93% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl rename to apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index e704a2c0c..549a0d870 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_clickhouse_SUITE). +-module(emqx_bridge_clickhouse_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -13,7 +13,7 @@ -include_lib("stdlib/include/assert.hrl"). -define(CLICKHOUSE_HOST, "clickhouse"). --define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse). +-define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector). %% This test SUITE requires a running clickhouse instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script @@ -21,7 +21,15 @@ %% from root of the EMQX directory.). You also need to set ?CLICKHOUSE_HOST and %% ?CLICKHOUSE_PORT to appropriate values. %% -%% docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" clickhouse/clickhouse-server +%% docker run \ +%% -d \ +%% -p 18123:8123 \ +%% -p 19000:9000 \ +%% --name some-clickhouse-server \ +%% --ulimit nofile=262144:262144 \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" \ +%% -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" \ +%% clickhouse/clickhouse-server all() -> emqx_common_test_helpers:all(?MODULE). @@ -119,7 +127,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, ResourceID}), QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, diff --git a/changes/ee/refactor-10654.en.md b/changes/ee/refactor-10654.en.md new file mode 100644 index 000000000..229d002da --- /dev/null +++ b/changes/ee/refactor-10654.en.md @@ -0,0 +1 @@ +The clickhouse bridge has been refactored so it is located in its own OTP application. diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index de5d8c3b1..7cbc30567 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -4,4 +4,3 @@ mongo_rs_sharded mysql redis redis_cluster -clickhouse diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 6e2dbcbce..ff2f8fed4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -17,7 +17,8 @@ emqx_bridge_rocketmq, emqx_bridge_rabbitmq, emqx_bridge_tdengine, - emqx_bridge_influxdb + emqx_bridge_influxdb, + emqx_bridge_clickhouse ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 17ffe9b9b..853247c1a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -32,9 +32,9 @@ api_schemas(Method) -> ref(emqx_bridge_timescale, Method), ref(emqx_bridge_matrix, Method), ref(emqx_bridge_tdengine, Method), - ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_bridge_dynamo, Method), ref(emqx_bridge_rocketmq, Method), + ref(emqx_bridge_clickhouse, Method), ref(emqx_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), ref(emqx_bridge_pulsar, Method ++ "_producer"), @@ -57,7 +57,7 @@ schema_modules() -> emqx_bridge_timescale, emqx_bridge_matrix, emqx_bridge_tdengine, - emqx_ee_bridge_clickhouse, + emqx_bridge_clickhouse, emqx_bridge_dynamo, emqx_bridge_rocketmq, emqx_bridge_sqlserver, @@ -101,7 +101,7 @@ resource_type(pgsql) -> emqx_connector_pgsql; resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_bridge_tdengine_connector; -resource_type(clickhouse) -> emqx_ee_connector_clickhouse; +resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(dynamo) -> emqx_bridge_dynamo_connector; resource_type(rocketmq) -> emqx_bridge_rocketmq_connector; resource_type(sqlserver) -> emqx_bridge_sqlserver_connector; @@ -297,7 +297,7 @@ clickhouse_structs() -> [ {clickhouse, mk( - hoconsc:map(name, ref(emqx_ee_bridge_clickhouse, "config")), + hoconsc:map(name, ref(emqx_bridge_clickhouse, "config")), #{ desc => <<"Clickhouse Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 3db090939..ef579c036 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,3 +1,2 @@ toxiproxy influxdb -clickhouse diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 3414c80b5..ee1d4e500 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 9a4f36cf3..e08413968 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -6,8 +6,7 @@ kernel, stdlib, ecpool, - hstreamdb_erl, - clickhouse + hstreamdb_erl ]}, {env, []}, {modules, []}, diff --git a/mix.exs b/mix.exs index 7b76bdc4b..28fa2c6cc 100644 --- a/mix.exs +++ b/mix.exs @@ -158,7 +158,6 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_gcp_pubsub, :emqx_bridge_cassandra, :emqx_bridge_opents, - :emqx_bridge_clickhouse, :emqx_bridge_dynamo, :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, @@ -175,7 +174,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_pulsar, :emqx_oracle, :emqx_bridge_oracle, - :emqx_bridge_rabbitmq + :emqx_bridge_rabbitmq, + :emqx_bridge_clickhouse ]) end diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse.hocon similarity index 98% rename from rel/i18n/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse.hocon index 6735aee22..726d1eb7c 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" diff --git a/rel/i18n/emqx_ee_connector_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse_connector.hocon similarity index 89% rename from rel/i18n/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/emqx_bridge_clickhouse_connector.hocon index cebba5aef..26746a52d 100644 --- a/rel/i18n/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" diff --git a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse.hocon similarity index 97% rename from rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse.hocon index a3ede08ba..ff291f667 100644 --- a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_clickhouse { +emqx_bridge_clickhouse { batch_value_separator.desc: """默认为逗号 ',',适用于 VALUE 格式。您也可以使用其他分隔符, 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" diff --git a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon similarity index 88% rename from rel/i18n/zh/emqx_ee_connector_clickhouse.hocon rename to rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon index f1457a1f6..b6fd7f3b7 100644 --- a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/zh/emqx_bridge_clickhouse_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_clickhouse { +emqx_bridge_clickhouse_connector { base_url.desc: """你想连接到的Clickhouse服务器的HTTP URL(例如http://myhostname:8123)。"""