From d126c7dc6265549eb0218065f99f5dbf3fe6dc02 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 28 Mar 2023 11:32:38 -0300 Subject: [PATCH 1/9] refactor(rule_engine): use more helper functions Follow up to https://github.com/emqx/emqx/pull/10251#discussion_r1150710899 and https://github.com/emqx/emqx/pull/10251#discussion_r1150720420 --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- .../src/emqx_rule_engine.app.src | 2 +- .../emqx_rule_engine/src/emqx_rule_engine.erl | 27 ++++++++++--------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 99a49f8fd..f5bcb23e2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 1681297ec..8d50f60e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index d494a4740..44b49a75b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -319,14 +319,8 @@ get_basic_usage_info() -> ReferencedBridges = lists:foldl( fun(#{actions := Actions, from := Froms}, Acc) -> - BridgeIDs0 = - [ - BridgeID - || From <- Froms, - {ok, BridgeID} <- - [emqx_bridge_resource:bridge_hookpoint_to_bridge_id(From)] - ], - BridgeIDs1 = lists:filter(fun is_binary/1, Actions), + BridgeIDs0 = get_referenced_hookpoints(Froms), + BridgeIDs1 = get_egress_bridges(Actions), tally_referenced_bridges(BridgeIDs0 ++ BridgeIDs1, Acc) end, #{}, @@ -490,10 +484,8 @@ forwards_to_bridge(Actions, BridgeId) -> lists:any(fun(A) -> A =:= BridgeId end, Actions). references_ingress_bridge(Froms, BridgeId) -> - lists:any( - fun(ReferenceBridgeId) -> - BridgeId =:= ReferenceBridgeId - end, + lists:member( + BridgeId, [ RefBridgeId || From <- Froms, @@ -501,3 +493,14 @@ references_ingress_bridge(Froms, BridgeId) -> [emqx_bridge_resource:bridge_hookpoint_to_bridge_id(From)] ] ). + +get_referenced_hookpoints(Froms) -> + [ + BridgeID + || From <- Froms, + {ok, BridgeID} <- + [emqx_bridge_resource:bridge_hookpoint_to_bridge_id(From)] + ]. + +get_egress_bridges(Actions) -> + lists:filter(fun is_binary/1, Actions). From d62698dd0b5a448599e86630088dceaf71fcd275 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Wed, 29 Mar 2023 16:32:18 +0800 Subject: [PATCH 2/9] chore: upgrade dashboard to e1.0.5-beta.4 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d4b5b5593..571fa4e01 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.2.0 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5-beta.3 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5-beta.4 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 624c168ddee871396fc75f4b85344835595e581b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 29 Mar 2023 11:39:18 +0200 Subject: [PATCH 3/9] fix: hide nodelay Kafka client socket option also fix a few descriptions --- .../i18n/emqx_ee_bridge_kafka.conf | 37 ++++++++++--------- .../src/emqx_ee_bridge_kafka.erl | 10 +++-- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 787a39fdb..651de99e9 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -216,6 +216,7 @@ emqx_ee_bridge_kafka { zh: "Socket 收包缓存大小" } } + # hidden socket_nodelay { desc { en: "When set to 'true', TCP buffer is sent as soon as possible. " @@ -571,26 +572,27 @@ emqx_ee_bridge_kafka { zh: "指定从哪个 Kafka 主题消费消息。" } label { - en: "Kafka topic" - zh: "Kafka 主题 " + en: "Kafka Topic" + zh: "Kafka 主题" } } consumer_max_batch_bytes { desc { - en: "Maximum bytes to fetch in a batch of messages." + en: "Set how many bytes to pull from Kafka in each fetch request. " "Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance." - zh: "在一批消息中要取的最大字节数。" + zh: "设置每次从 Kafka 拉取数据的字节数。" "如果该配置小于 Kafka 中消息到大小，则可能会影响消费性能。" } label { - en: "Max Bytes" - zh: "最大字节数" + en: "Fetch Bytes" + zh: "拉取字节数" } } + # hidden consumer_max_rejoin_attempts { desc { en: "Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay." - zh: "允许一个成员重新加入小组的最大次数。如果超过改配置次数后仍不能成功加入消费组，则会在延迟一段时间后再重试。" + zh: "允许一个消费组成员重新加入小组的最大次数。如果超过改配置次数后仍不能成功加入消费组，则会在延迟一段时间后再重试。" } label { en: "Max Rejoin Attempts" @@ -599,10 +601,9 @@ emqx_ee_bridge_kafka { } consumer_offset_reset_policy { desc { - en: "Defines from which offset a consumer should start fetching when there" - " is no commit history or when the commit history becomes invalid." - zh: "当没有主题分区没有偏移量的历史记录，或则历史记录失效后，" - "消费者应该使用哪个偏移量重新开始消费" + en: "Defines from which offset a consumer should start fetching when there " + "is no commit history or when the commit history becomes invalid." + zh: "当没有偏移量的历史记录，或则历史记录失效后，消费者应该使用哪个偏移量重新开始消费。" } label { en: "Offset Reset Policy" @@ -616,13 +617,13 @@ emqx_ee_bridge_kafka { } label { en: "Offset Commit Interval" - zh: "偏移承诺间隔" + zh: "偏移提交间隔" } } consumer_topic_mapping { desc { - en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." - zh: "指定 Kafka 主题和 MQTT 主题之间的映射。 必须至少包含一个项目。" + en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." + zh: "指定 Kafka 主题和 MQTT 主题之间的映射关系。 必须至少包含一项。" } label { en: "Topic Mapping" @@ -632,9 +633,9 @@ emqx_ee_bridge_kafka { consumer_key_encoding_mode { desc { en: "Defines how the key from the Kafka message is" - " dealt with before being forwarded via MQTT.\n" + " encoded before being forwarded via MQTT.\n" "none Uses the key from the Kafka message unchanged." - " Note: in this case, then the key must be a valid UTF-8 string.\n" + " Note: in this case, the key must be a valid UTF-8 string.\n" "base64 Uses base-64 encoding on the received key." zh: "定义了在通过MQTT转发之前如何处理Kafka消息的 Key。" "none 使用Kafka消息中的 Key 原始值，不进行编码。" @@ -649,9 +650,9 @@ emqx_ee_bridge_kafka { consumer_value_encoding_mode { desc { en: "Defines how the value from the Kafka message is" - " dealt with before being forwarded via MQTT.\n" + " encoded before being forwarded via MQTT.\n" "none Uses the value from the Kafka message unchanged." - " Note: in this case, then the value must be a valid UTF-8 string.\n" + " Note: in this case, the value must be a valid UTF-8 string.\n" "base64 Uses base-64 encoding on the received value." zh: "定义了在通过MQTT转发之前如何处理Kafka消息的 Value。" "none 使用Kafka消息中的 Value 原始值，不进行编码。" diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index f623417b2..4a9263134 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -221,17 +221,21 @@ fields(socket_opts) -> {sndbuf, mk( emqx_schema:bytesize(), - #{default => <<"1024KB">>, desc => ?DESC(socket_send_buffer)} + #{default => <<"1MB">>, desc => ?DESC(socket_send_buffer)} )}, {recbuf, mk( emqx_schema:bytesize(), - #{default => <<"1024KB">>, desc => ?DESC(socket_receive_buffer)} + #{default => <<"1MB">>, desc => ?DESC(socket_receive_buffer)} )}, {nodelay, mk( boolean(), - #{default => true, desc => ?DESC(socket_nodelay)} + #{ + default => true, + hidden => true, + desc => ?DESC(socket_nodelay) + } )} ]; fields(producer_opts) -> From b2b9732edbfe4af80b67bbcda3f65ea6c63abd8f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 29 Mar 2023 09:30:06 -0300 Subject: [PATCH 4/9] fix(tdengine): make password a required field (rv5.0) Fixes https://emqx.atlassian.net/browse/EMQX-9395 ``` 2023-03-29T09:59:02.655495+00:00 [warning] msg: start_resource_failed, mfa: emqx_resource_manager:start_resource/2, line: 524, id: <<"bridge:tdengine:jimmoen-test-trigger">>, reason: {error,function_clause,[{emqx_ee_connector_tdengine,on_start,[<<"bridge:tdengine:jimmoen-test-trigger:1076">>,#{database => <<"db">>,enable => true,pool_size => 8,resource_opts => #{auto_restart_interval => 60000,batch_size => 1,batch_time => 0,health_check_interval => 15000,max_queue_bytes => 104857600,query_mode => sync,request_timeout => 15000,start_after_created => true,start_timeout => 5000,worker_pool_size => 16},server => "127.0.0.1",sql => <<"insert into mqtt.t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})">>,username => <<"root">>}],[{file,"emqx_ee_connector_tdengine.erl"},{line,76}]},{emqx_resource,call_start,3,[{file,"emqx_resource.erl"},{line,359}]},{emqx_resource_manager,start_resource,2,[{file,"emqx_resource_manager.erl"},{line,513}]},{gen_statem,loop_state_callback,11,[{file,"gen_statem.erl"},{line,1205}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]} ``` --- .ci/docker-compose-file/docker-compose.yaml | 3 +++ .../emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- .../src/emqx_ee_connector_tdengine.erl | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 5c37d971e..526c4c182 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -16,6 +16,9 @@ services: GITHUB_REF: ${GITHUB_REF} networks: - emqx_bridge + ports: + - 28083:18083 + - 2883:1883 volumes: - ../..:/emqx - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 6f40f7158..a005071da 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 7ab0c5078..746ab5485 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -41,14 +41,16 @@ roots() -> fields(config) -> [ {server, server()} - | add_default_username(emqx_connector_schema_lib:relational_db_fields()) + | adjust_fields(emqx_connector_schema_lib:relational_db_fields()) ]. -add_default_username(Fields) -> +adjust_fields(Fields) -> lists:map( fun ({username, OrigUsernameFn}) -> {username, add_default_fn(OrigUsernameFn, <<"root">>)}; + ({password, OrigPasswordFn}) -> + {password, make_required_fn(OrigPasswordFn)}; (Field) -> Field end, @@ -61,6 +63,12 @@ add_default_fn(OrigFn, Default) -> (Field) -> OrigFn(Field) end. +make_required_fn(OrigFn) -> + fun + (required) -> true; + (Field) -> OrigFn(Field) + end. + server() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?TD_HOST_OPTIONS). From 94726eb7f18a571b9afe974df8b7e7a5880c529a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 29 Mar 2023 15:05:32 +0200 Subject: [PATCH 5/9] docs: refine zh translations --- .../i18n/emqx_ee_bridge_kafka.conf | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 651de99e9..df32c1cae 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -581,7 +581,7 @@ emqx_ee_bridge_kafka { en: "Set how many bytes to pull from Kafka in each fetch request. " "Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance." zh: "设置每次从 Kafka 拉取数据的字节数。" - "如果该配置小于 Kafka 中消息到大小，则可能会影响消费性能。" + "如该配置小于 Kafka 消息的大小，可能会影响消费性能。" } label { en: "Fetch Bytes" @@ -592,7 +592,7 @@ emqx_ee_bridge_kafka { consumer_max_rejoin_attempts { desc { en: "Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay." - zh: "允许一个消费组成员重新加入小组的最大次数。如果超过改配置次数后仍不能成功加入消费组，则会在延迟一段时间后再重试。" + zh: "消费组成员允许重新加入小组的最大次数。如超过该配置次数后仍未能成功加入消费组，则会在等待一段时间后重试。" } label { en: "Max Rejoin Attempts" @@ -603,7 +603,7 @@ emqx_ee_bridge_kafka { desc { en: "Defines from which offset a consumer should start fetching when there " "is no commit history or when the commit history becomes invalid." - zh: "当没有偏移量的历史记录，或则历史记录失效后，消费者应该使用哪个偏移量重新开始消费。" + zh: "如不存在偏移量历史记录或历史记录失效，消费者应使用哪个偏移量开始消费。" } label { en: "Offset Reset Policy" @@ -623,7 +623,7 @@ emqx_ee_bridge_kafka { consumer_topic_mapping { desc { en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." - zh: "指定 Kafka 主题和 MQTT 主题之间的映射关系。 必须至少包含一项。" + zh: "指定 Kafka 主题和 MQTT 主题之间的映射关系。 应至少包含一项。" } label { en: "Topic Mapping" @@ -637,10 +637,10 @@ emqx_ee_bridge_kafka { "none Uses the key from the Kafka message unchanged." " Note: in this case, the key must be a valid UTF-8 string.\n" "base64 Uses base-64 encoding on the received key." - zh: "定义了在通过MQTT转发之前如何处理Kafka消息的 Key。" - "none 使用Kafka消息中的 Key 原始值，不进行编码。" - " 注意：在这种情况下，Key 必须是一个有效的UTF-8字符串。\n" - "base64 对收到的密钥或值使用base-64编码。" + zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Key。" + "none 使用 Kafka 消息中的 Key 原始值，不进行编码。" + " 注意：在这种情况下，Key 必须是一个有效的 UTF-8 字符串。\n" + "base64 对收到的密钥或值使用 base-64 编码。" } label { en: "Key Encoding Mode" @@ -654,10 +654,10 @@ emqx_ee_bridge_kafka { "none Uses the value from the Kafka message unchanged." " Note: in this case, the value must be a valid UTF-8 string.\n" "base64 Uses base-64 encoding on the received value." - zh: "定义了在通过MQTT转发之前如何处理Kafka消息的 Value。" - "none 使用Kafka消息中的 Value 原始值，不进行编码。" - " 注意：在这种情况下，Value 必须是一个有效的UTF-8字符串。\n" - "base64 对收到的 Value 使用base-64编码。" + zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Value。" + "none 使用 Kafka 消息中的 Value 原始值，不进行编码。" + " 注意：在这种情况下，Value 必须是一个有效的 UTF-8 字符串。\n" + "base64 对收到的 Value 使用 base-64 编码。" } label { en: "Value Encoding Mode" From 806ee716ee0b72684dc2def56bfe136a65f7562d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 29 Mar 2023 16:41:42 -0300 Subject: [PATCH 6/9] fix(redis): remove `required` meta from schema Since there's a default value, this field shouldn't be marked as required. --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_connector/src/emqx_connector_redis.erl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 4f7ff0f25..08f97c243 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 29f7c0a38..c70e766af 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -285,7 +285,6 @@ redis_fields() -> {database, #{ type => integer(), default => 0, - required => true, desc => ?DESC("database") }}, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} From 8c4d211b547ece550899a24e07f92e2c8b1d8ec7 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 11:17:04 +0200 Subject: [PATCH 7/9] fix: on_get_status clickhouse just reporting connecting The on_get_status callback for clickhouse just returned `connecting` without error information when the status check was unsuccessful. This is fixed by letting the callback return error information similarly to how the HTTP connector does it. Fixes: https://emqx.atlassian.net/browse/EMQX-9374 https://emqx.atlassian.net/browse/EMQX-9278 --- .../i18n/emqx_ee_connector_clickhouse.conf | 11 +++ lib-ee/emqx_ee_connector/rebar.config | 2 +- .../src/emqx_ee_connector_clickhouse.erl | 71 +++++++++++++++---- .../test/ee_connector_clickhouse_SUITE.erl | 3 +- 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf index 1e07c29b4..533d100bf 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf @@ -12,4 +12,15 @@ emqx_ee_connector_clickhouse { } } + connect_timeout { + desc { + en: "The timeout when connecting to the Clickhouse server." + zh: "连接HTTP服务器的超时时间。" + } + label: { + en: "Clickhouse Timeout" + zh: "连接超时" + } + } + } diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 76f6ccfba..4c1797e34 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -3,7 +3,7 @@ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.2"}}}, + {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index b1ad6c787..8f2fdc042 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -53,7 +53,6 @@ %% Internal exports used to execute code with ecpool worker -export([ - check_database_status/1, execute_sql_in_clickhouse_server_using_connection/2 ]). @@ -102,6 +101,14 @@ fields(config) -> end, desc => ?DESC("base_url") } + )}, + {connect_timeout, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"15s">>, + desc => ?DESC("connect_timeout") + } )} ] ++ emqx_connector_schema_lib:relational_db_fields(). @@ -137,7 +144,8 @@ on_start( #{ url := URL, database := DB, - pool_size := PoolSize + pool_size := PoolSize, + connect_timeout := ConnectTimeout } = Config ) -> ?SLOG(info, #{ @@ -155,7 +163,10 @@ on_start( {pool_size, PoolSize}, {pool, PoolName} ], - InitState = #{poolname => PoolName}, + InitState = #{ + poolname => PoolName, + connect_timeout => ConnectTimeout + }, try Templates = prepare_sql_templates(Config), State = maps:merge(InitState, #{templates => Templates}), @@ -282,18 +293,52 @@ on_stop(ResourceID, #{poolname := PoolName}) -> %% on_get_status emqx_resouce callback and related functions %% ------------------------------------------------------------------- -on_get_status(_ResourceID, #{poolname := Pool} = _State) -> - case - emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:check_database_status/1) - of - true -> - connected; - false -> - connecting +on_get_status( + _InstId, + #{ + poolname := PoolName, + connect_timeout := Timeout + } = State +) -> + case do_get_status(PoolName, Timeout) of + ok -> + {connected, State}; + {error, Reason} -> + {disconnected, State, Reason} end. -check_database_status(Connection) -> - clickhouse:status(Connection). +do_get_status(PoolName, Timeout) -> + Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + DoPerWorker = + fun(Worker) -> + case ecpool_worker:exec(Worker, fun clickhouse:detailed_status/1, Timeout) of + ok -> + ok; + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "clickhouse_connector_get_status_failed", + reason => Reason, + worker => Worker + }), + Error + end + end, + try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + Results -> + case [E || {error, _} = E <- Results] of + [] -> + ok; + Errors -> + hd(Errors) + end + catch + exit:timeout -> + ?SLOG(error, #{ + msg => "clickhouse_connector_pmap_failed", + reason => timeout + }), + {error, timeout} + end. %% ------------------------------------------------------------------- %% on_query emqx_resouce callback and related functions diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl index eab1aa054..111deba06 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl @@ -190,7 +190,8 @@ clickhouse_config() -> ?CLICKHOUSE_DEFAULT_PORT ] ) - ) + ), + connect_timeout => 10000 }, #{<<"config">> => Config}. From 2755829ebff362846c8fcffd3386fe1d5b29680c Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 14:43:13 +0200 Subject: [PATCH 8/9] docs: add changelog entry for clickhouse fix --- changes/ee/fix-10289.en.md | 1 + changes/ee/fix-10289.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/fix-10289.en.md create mode 100644 changes/ee/fix-10289.zh.md diff --git a/changes/ee/fix-10289.en.md b/changes/ee/fix-10289.en.md new file mode 100644 index 000000000..65eed7b5d --- /dev/null +++ b/changes/ee/fix-10289.en.md @@ -0,0 +1 @@ +Clickhouse has got a fix that makes the error message better when users click the test button in the settings dialog. diff --git a/changes/ee/fix-10289.zh.md b/changes/ee/fix-10289.zh.md new file mode 100644 index 000000000..d47278c16 --- /dev/null +++ b/changes/ee/fix-10289.zh.md @@ -0,0 +1 @@ +Clickhouse 已经修复了一个问题，当用户在设置对话框中点击测试按钮时，错误信息会更清晰。 From a0361e1021bcae61046f0a188754b71ec5d01b80 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 31 Mar 2023 17:05:41 +0800 Subject: [PATCH 9/9] chore: upgrade dashboard to e1.0.5 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 571fa4e01..410140616 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.2.0 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5-beta.4 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT)