From 27aa3f49aa64a3691218d67529c9e0f8e938657e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Mar 2023 15:02:01 +0800 Subject: [PATCH 01/50] docs: improve some mqtt options description --- apps/emqx/i18n/emqx_schema_i18n.conf | 106 ++++++++++++++------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 6f926ec39..c14ae3f61 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -666,15 +666,15 @@ mqtt 下所有的配置作为全局的默认值存在，它可以被 zone< mqtt_idle_timeout { desc { - en: """After the TCP connection is established, if the MQTT CONNECT packet from the client is -not received within the time specified by idle_timeout, the connection will be disconnected. -After the CONNECT packet has been accepted by EMQX, if the connection idles for this long time, -then the Erlang process is put to hibernation to save OS resources. Note: long idle_timeout -interval may impose risk at the system if large number of malicious clients only establish connections -but do not send any data.""" - zh: """TCP 连接建立后，如果在 idle_timeout 指定的时间内未收到客户端的 MQTT CONNECT 报文，则连接将被断开。 -如果连接在 CONNECT 报文被 EMQX 接受之后空闲超过该时长，那么服务这个连接的 Erlang 进程会进入休眠以节省系统资源。 -注意，该配置值如果设置过大的情况下，如果大量恶意客户端只连接，但不发任何数据，可能会导致系统资源被恶意消耗。""" + en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: + - Automatically disconnected if no CONNECT package is received from the client yet. + - Put into hibernation mode to save resources if some CONNECT packages are already received. +Note: Please set the parameter with caution as long idle time will lead to resouce waste.""" + zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， + - 如暂未收到客户端的 CONNECT 报文，连接将断开； + - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 + +注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" } label: { en: """Idle Timeout""" @@ -783,8 +783,8 @@ but do not send any data.""" mqtt_ignore_loop_deliver { desc { - en: """Ignore loop delivery of messages for MQTT v3.1.1/v3.1.0, similar to No Local subscription option in MQTT 5.0.""" - zh: """是否为 MQTT v3.1.1/v3.1.0 客户端忽略投递自己发布的消息，类似于 MQTT 5.0 中的 No Local 订阅选项。""" + en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be forwarded to the client itself, similar to No Local in MQTT 5.0.""" + zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" } label: { en: """Ignore Loop Deliver""" @@ -794,10 +794,10 @@ but do not send any data.""" mqtt_strict_mode { desc { - en: """Parse MQTT messages in strict mode. -When set to true, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected""" + en: """Whether to parse MQTT messages in strict mode. +In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" zh: """是否以严格模式解析 MQTT 消息。 -当设置为 true 时，例如客户端 ID、主题名称等中的无效 utf8 字符串将导致客户端断开连接。""" +严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" } label: { en: """Strict Mode""" @@ -807,8 +807,10 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_response_information { desc { - en: """Specify the response information returned to the client. This feature is disabled if is set to \"\". Applies only to clients using MQTT 5.0.""" - zh: """指定返回给客户端的响应信息。如果设置为 \"\"，则禁用此功能。仅适用于使用 MQTT 5.0 协议的客户端。""" + en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communite under the topic reqrsp/. +To disable this feature, input \"\" in the text box below.""" + zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 +如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" } label: { en: """Response Information""" @@ -818,23 +820,23 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_server_keepalive { desc { - en: """The keep alive that EMQX requires the client to use. If configured as disabled, it means that the keep alive specified by the client will be used. Requires Server Keep Alive in MQTT 5.0, so it is only applicable to clients using MQTT 5.0 protocol.""" - zh: """EMQX 要求客户端使用的保活时间，配置为 disabled 表示将使用客户端指定的保活时间。需要用到 MQTT 5.0 中的 Server Keep Alive，因此仅适用于使用 MQTT 5.0 协议的客户端。""" + en: """The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" + zh: """EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" } label: { en: """Server Keep Alive""" - zh: """服务端保持连接""" + zh: """服务端保活时间""" } } mqtt_keepalive_backoff { desc { - en: """The backoff multiplier used by the broker to determine the client keep alive timeout. If EMQX doesn't receive any packet in Keep Alive * Backoff * 2 seconds, EMQX will close the current connection.""" - zh: """Broker 判定客户端保活超时使用的退避乘数。如果 EMQX 在 Keep Alive * Backoff * 2 秒内未收到任何报文，EMQX 将关闭当前连接。""" + en: """The coeffient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" } label: { en: """Keep Alive Backoff""" - zh: """保持连接退避乘数""" + zh: """保活超时阈值系数""" } } @@ -978,14 +980,14 @@ To configure \"topic/1\" > \"topic/2\": mqtt_use_username_as_clientid { desc { - en: """Whether to user Client ID as Username. -This setting takes effect later than Use Peer Certificate as Username (peer_cert_as_username) and Use peer certificate as Client ID (peer_cert_as_clientid).""" + en: """Whether to use Username as Client ID. +This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" zh: """是否使用用户名作为客户端 ID。 -此设置的作用时间晚于 使用对端证书作为用户名（peer_cert_as_username） 和 使用对端证书作为客户端 ID（peer_cert_as_clientid）。""" +此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" } label: { en: """Use Username as Client ID""" - zh: """使用用户名作为客户端 ID""" + zh: """用户名作为客户端 ID""" } } @@ -993,22 +995,22 @@ This setting takes effect later than Use Peer Certificate as Usernamecn: Take the CN field of the certificate as Username -- dn: Take the DN field of the certificate as Username -- crt: Take the content of the DER or PEM certificate as Username -- pem: Convert DER certificate content to PEM format as Username -- md5: Take the MD5 value of the content of the DER or PEM certificate as Username""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名。仅适用于 TLS 连接。 -目前支持配置为以下内容： -- cn: 取证书的 CN 字段作为 Username -- dn: 取证书的 DN 字段作为 Username -- crt: 取 DER 或 PEM 证书的内容作为 Username -- pem: 将 DER 证书内容转换为 PEM 格式后作为 Username -- md5: 取 DER 或 PEM 证书的内容的 MD5 值作为 Username""" +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: Content of the DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Username +- md5: MD5 value of the DER or PEM certificate""" + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 的证书内容 +- pem: 将 DER 证书转换为 PEM 格式作为用户名 +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" } label: { en: """Use Peer Certificate as Username""" - zh: """使用对端证书作为用户名""" + zh: """对端证书作为用户名""" } } @@ -1016,22 +1018,22 @@ Supported configurations are the following: desc { en: """Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. Supported configurations are the following: -- cn: Take the CN field of the certificate as Client ID -- dn: Take the DN field of the certificate as Client ID -- crt: Take the content of the DER or PEM certificate as Client ID -- pem: Convert DER certificate content to PEM format as Client ID -- md5: Take the MD5 value of the content of the DER or PEM certificate as Client ID""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接。 -目前支持配置为以下内容： -- cn: 取证书的 CN 字段作为 Client ID -- dn: 取证书的 DN 字段作为 Client ID -- crt: 取 DER 或 PEM 证书的内容作为 Client ID -- pem: 将 DER 证书内容转换为 PEM 格式后作为 Client ID -- md5: 取 DER 或 PEM 证书的内容的 MD5 值作为 Client ID""" +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Client ID +- md5: MD5 value of the DER or PEM certificate""" + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 证书的内容 +- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" } label: { en: """Use Peer Certificate as Client ID""" - zh: """使用对端证书作为客户端 ID""" + zh: """对端证书作为客户端 ID""" } } From 758e61005996fdde7c4463507164d5f46d77fd84 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 08:42:08 +0800 Subject: [PATCH 02/50] chore: apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index c14ae3f61..1d8c100d0 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -783,8 +783,8 @@ Note: Please set the parameter with caution as long idle time will lead to resou mqtt_ignore_loop_deliver { desc { - en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be forwarded to the client itself, similar to No Local in MQTT 5.0.""" - zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" + en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" + zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" } label: { en: """Ignore Loop Deliver""" @@ -807,7 +807,7 @@ In strict mode, invalid utf8 strings in for example client ID, topic name, etc. mqtt_response_information { desc { - en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communite under the topic reqrsp/. + en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. To disable this feature, input \"\" in the text box below.""" zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" From 3c495f8fd2f57af25d54ba5c93f6d26f92af43cb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 08:44:28 +0800 Subject: [PATCH 03/50] chore: update apps/emqx/i18n/emqx_schema_i18n.conf --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 1d8c100d0..5992f2c5d 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -808,7 +808,7 @@ In strict mode, invalid utf8 strings in for example client ID, topic name, etc. mqtt_response_information { desc { en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. -To disable this feature, input \"\" in the text box below.""" +To disable this feature, input \"\" in the text box below. Only applicable to MQTT 5.0 clients.""" zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" } From ee5b9d7e0a893f7c401e98f6494a147482bba55c Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 22 Mar 2023 13:56:47 +0800 Subject: [PATCH 04/50] fix(tdengine): remove the redundant table name in SQL template --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index b72d79955..f031cbfbf 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -22,7 +22,7 @@ ]). -define(DEFAULT_SQL, << - "insert into mqtt.t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) " + "insert into t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) " "values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})" >>). From 1ff8ecf60455f9ba4107f3ad994af98e504ddb95 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:10:10 +0800 Subject: [PATCH 05/50] chore: fix typos --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 5992f2c5d..dd53be19f 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -669,7 +669,7 @@ mqtt 下所有的配置作为全局的默认值存在，它可以被 zone< en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: - Automatically disconnected if no CONNECT package is received from the client yet. - Put into hibernation mode to save resources if some CONNECT packages are already received. -Note: Please set the parameter with caution as long idle time will lead to resouce waste.""" +Note: Please set the parameter with caution as long idle time will lead to resource waste.""" zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， - 如暂未收到客户端的 CONNECT 报文，连接将断开； - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 @@ -831,7 +831,7 @@ To disable this feature, input \"\" in the text box below. Only app mqtt_keepalive_backoff { desc { - en: """The coeffient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + en: """The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" } label: { From dfc9141da66ddf520f78d23b430eba24a2b91776 Mon Sep 17 00:00:00 2001 From: Meggielqk <126552073+Meggielqk@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:18:55 +0800 Subject: [PATCH 06/50] docs: Refined Chinese UI Also updated the English hints according to the refined Chinese according to Guowei's comments. Co-authored-by: Kjell Winblad --- .../i18n/emqx_ee_bridge_clickhouse.conf | 36 +++---------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf index 6a28b371a..5c6ae05f2 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf @@ -19,17 +19,8 @@ will be forwarded. } sql_template { desc { - en: """SQL Template. The template string can contain placeholders -for message metadata and payload field. The placeholders are inserted -without any checking and special formatting, so it is important to -ensure that the inserted values are formatted and escaped correctly.""" - zh: - """SQL模板。模板字符串可以包含消息元数据和有效载荷字段的占位符。占位符 -的插入不需要任何检查和特殊格式化，因此必须确保插入的数值格式化和转义正确。模板字符串可以包含占位符 -模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入 -所以必须确保插入的值的格式正确。因此，确保插入的值格式化和转义正确是非常重要的。模板字符串可以包含占位符 -模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入 -所以必须确保插入的值的格式正确。确保插入的值被正确地格式化和转义。""" + en: """The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported.""" + zh: """可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。""" } label { en: "SQL Template" @@ -38,29 +29,12 @@ ensure that the inserted values are formatted and escaped correctly.""" } batch_value_separator { desc { - en: """The bridge repeats what comes after the VALUES or FORMAT FormatType in the -SQL template to form a batch request. The value specified with -this parameter will be inserted between the values. The default -value ',' works for the VALUES format, but other values -might be needed if you specify some other format with the -clickhouse FORMAT syntax. - -See https://clickhouse.com/docs/en/sql-reference/statements/insert-into/ and -https://clickhouse.com/docs/en/interfaces/formats#formats for more information about -the format syntax and the available formats.""" - zh: """桥接会重复 VALUES 或 FORMAT 格式类型之后的内容。中 VALUES 或 -FORMAT FormatType 后面的内容，以形成一个批处理请求。用这个参数指定的值 -这个参数指定的值将被插入到这些值之间。默认的 -默认值','适用于VALUES格式，但是如果你指定了其他的格式，可能需要其他的值。可能需要其他值，如果你用 -"clickhouse FORMAT "语法指定其他格式。语法指定其他格式。 - -参见https://clickhouse.com/docs/en/sql-reference/statements/insert-into/ 和 -https://clickhouse.com/docs/en/interfaces/formats#formats 了解更多关于 -格式语法和可用的格式。""" + en: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" + zh: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" } label { en: "Batch Value Separator" - zh: "批量值分离器" + zh: "批量值分隔符" } } config_enable { From aefa0e5ffb05ac2c66c1b48f91d6100e3c8a9eb5 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 15:06:15 +0200 Subject: [PATCH 07/50] docs: make clickhouse config label for Server URL more concise --- .../emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf index 1e07c29b4..b3c2f6532 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf @@ -7,8 +7,8 @@ emqx_ee_connector_clickhouse { zh: """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" } label: { - en: "URL to clickhouse server" - zh: "到clickhouse服务器的URL" + en: "Server URL" + zh: "服务器 URL" } } From 91a784134c15ea56bdd7dae04307187633364da4 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 15:32:30 +0200 Subject: [PATCH 08/50] docs: update Chinese version of batch value separator label --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf index 6a28b371a..5096f8590 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf @@ -60,7 +60,7 @@ https://clickhouse.com/docs/en/interfaces/formats#formats 了解更多关于 } label { en: "Batch Value Separator" - zh: "批量值分离器" + zh: "分隔符" } } config_enable { From 632bffd45168bfb5ab8ab1e4b5f020b312de7fc8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 29 Mar 2023 11:59:29 -0300 Subject: [PATCH 09/50] fix: return friendly message when kafka producer fails to start (rv5.0) Fixes https://emqx.atlassian.net/browse/EMQX-9392 The returned information does not allow to diagnose the issue (i.e.: a connection issue due to the wrong host and port, the wrong password failing authn). However, such information is printed to the logs. This changes the returned error to the API so that the user is hinted at looking at the logs for further investigation of the error. --- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_resource/src/emqx_resource.erl | 9 ++++++++- .../src/kafka/emqx_bridge_impl_kafka_consumer.erl | 6 +++++- .../src/kafka/emqx_bridge_impl_kafka_producer.erl | 6 +++++- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 12 +++++++++--- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 7be1bcb1c..fbfe8c1fa 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 1ccb5ca71..a2fd9804b 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -356,7 +356,14 @@ is_buffer_supported(Module) -> -spec call_start(manager_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(MgrId, Mod, Config) -> - ?SAFE_CALL(Mod:on_start(MgrId, Config)). + try + Mod:on_start(MgrId, Config) + catch + throw:{error, Error} -> + {error, Error}; + Kind:Error:Stacktrace -> + {error, {Kind, Error, Stacktrace}} + end. -spec call_health_check(manager_id(), module(), resource_state()) -> resource_status() diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index a05f6ec13..44ea95f90 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -152,7 +152,11 @@ on_start(InstanceId, Config) -> kafka_hosts => BootstrapHosts, reason => emqx_misc:redact(Reason) }), - throw(failed_to_start_kafka_client) + throw( + {error, + "Failed to start Kafka client. Please check the logs for errors and check" + " the connection parameters"} + ) end, start_consumer(Config, InstanceId, ClientID). diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index 5703c69f5..8f5988aaf 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -114,7 +114,11 @@ on_start(InstId, Config) -> client_id => ClientId } ), - throw(failed_to_start_kafka_producer) + throw( + {error, + "Failed to start Kafka client. Please check the logs for errors and check" + " the connection parameters"} + ) end. on_stop(_InstanceID, #{client_id := ClientID, producers := Producers, resource_id := ResourceID}) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 4b9642442..2fb6c6857 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -9,6 +9,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). -define(PRODUCER, emqx_bridge_impl_kafka_producer). @@ -415,9 +416,14 @@ t_failed_creation_then_fix(Config) -> Type, erlang:list_to_atom(Name), WrongConf ), WrongConfigAtom = WrongConfigAtom1#{bridge_name => Name}, - ?assertThrow(failed_to_start_kafka_producer, ?PRODUCER:on_start(ResourceId, WrongConfigAtom)), - %% before throwing, it should cleanup the client process. - ?assertEqual([], supervisor:which_children(wolff_client_sup)), + ?assertThrow( + {error, _}, + ?PRODUCER:on_start(ResourceId, WrongConfigAtom) + ), + %% before throwing, it should cleanup the client process. we + %% retry because the supervisor might need some time to really + %% remove it from its tree. + ?retry(50, 10, ?assertEqual([], supervisor:which_children(wolff_client_sup))), %% must succeed with correct config {ok, #{config := ValidConfigAtom1}} = emqx_bridge:create( Type, erlang:list_to_atom(Name), ValidConf From 7c597bfaa91092fa5050f0ad167b26de9371896e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 30 Mar 2023 16:05:32 +0200 Subject: [PATCH 10/50] docs: give a better idea of what this application is supposed to do --- apps/emqx_management/README.md | 46 ++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index fa37d0f0f..2edeae91a 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -1,12 +1,44 @@ -# emqx-management +# EMQX Management -EMQX Management API +EMQX Management offers various interfaces for administrators to interact with +the system, either by a remote console attached to a running node, a CLI (i.e. +`./emqx ctl`), or through its rich CRUD-style REST API (mostly used by EMQX' +dashboard). The system enables administrators to modify both cluster and +individual node configurations, and provides the ability to view and reset +different statistics and metrics. -## How to Design RESTful API? +## Functionality -http://restful-api-design.readthedocs.io/en/latest/scope.html +Amongst others it allows to manage -default application see: -header: -authorization: Basic YWRtaW46cHVibGlj +* alarms, +* API Keys, +* banned clients, users or hosts, +* connected clients including their topic subscriptions, +* cluster configurations, +* configuration of MQTT listeners, +* node configuration, +* custom plugins, +* fixed subscriptions, +* and topics. + +Moreover it lets you + +* modify hot and non-hot updatable configuration values, +* publish messages, as well as bulk messages, +* create trace files, +* and last but not least monitor system status. + +## Implementation Notes + +API endpoints are implemented using the `minirest` framework in combination with +`hoconsc` and `emqx_dashboard_swagger`. + +## TODO/FIXME + +At its current state there are some reverse dependencies from other applications +that do calls directly into `emqx_mgmt`. + +Also, and somewhat related, its bpapi proto modules do calls directly into +other applications. From fec0e7ab0f591602d500e9cacb049dffc1387ac8 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 22 Mar 2023 14:17:45 +0800 Subject: [PATCH 11/50] chore: update changes --- changes/ee/fix-10201.en.md | 1 + changes/ee/fix-10201.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/fix-10201.en.md create mode 100644 changes/ee/fix-10201.zh.md diff --git a/changes/ee/fix-10201.en.md b/changes/ee/fix-10201.en.md new file mode 100644 index 000000000..b3dd53150 --- /dev/null +++ b/changes/ee/fix-10201.en.md @@ -0,0 +1 @@ +In TDengine, removed the redundant database name from the SQL template. diff --git a/changes/ee/fix-10201.zh.md b/changes/ee/fix-10201.zh.md new file mode 100644 index 000000000..53b175551 --- /dev/null +++ b/changes/ee/fix-10201.zh.md @@ -0,0 +1 @@ +在 TDengine 桥接的 SQL 模板中，删除了多余的数据库表名。 From 1ff96f5314dfc83051e361c27444cadeaf02fc00 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 31 Mar 2023 09:18:39 +0200 Subject: [PATCH 12/50] style: fix wording Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_management/README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index 2edeae91a..9378d18b1 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -12,16 +12,13 @@ different statistics and metrics. Amongst others it allows to manage -* alarms, -* API Keys, -* banned clients, users or hosts, -* connected clients including their topic subscriptions, -* cluster configurations, -* configuration of MQTT listeners, -* node configuration, -* custom plugins, -* fixed subscriptions, -* and topics. +* Alarms +* API Keys +* Banned clients, users or hosts +* Clients (and sessions) including their topic subscriptions +* Configurations +* Manage plugins +* Fixed subscriptions Moreover it lets you @@ -33,7 +30,7 @@ Moreover it lets you ## Implementation Notes API endpoints are implemented using the `minirest` framework in combination with -`hoconsc` and `emqx_dashboard_swagger`. +HOCON schema and OpenAPI 3.0 specifications. ## TODO/FIXME From 7e31e60e90f358fbbc1d46fdfceb6270cb655f1b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 31 Mar 2023 09:19:48 +0200 Subject: [PATCH 13/50] style: fix wording --- apps/emqx_management/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index 9378d18b1..aa5d0c606 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -19,6 +19,7 @@ Amongst others it allows to manage * Configurations * Manage plugins * Fixed subscriptions +* Topics Moreover it lets you From 6f6d12f930cfbff084b4ce62184e9c9d20cb0919 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 31 Mar 2023 09:39:04 +0200 Subject: [PATCH 14/50] docs: better Chinese labels Co-authored-by: LenaLenaPan <120552185+LenaLenaPan@users.noreply.github.com> --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf index 5c6ae05f2..4cfb4df00 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf @@ -34,7 +34,7 @@ will be forwarded. } label { en: "Batch Value Separator" - zh: "批量值分隔符" + zh: "分隔符" } } config_enable { From bcde52383b7166e9bfc4683ec651122da1d42c0a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 31 Mar 2023 12:35:27 +0200 Subject: [PATCH 15/50] docs: fix max batch size desc --- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index fb6b2eb06..600289b1d 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -149,11 +149,11 @@ When disabled the messages are buffered in RAM only.""" batch_size { desc { en: """Maximum batch count. If equal to 1, there's effectively no batching.""" - zh: """批量请求大小。如果设为1，则无批处理。""" + zh: """最大批量请求大小。如果设为1，则无批处理。""" } label { - en: """Batch size""" - zh: """批量请求大小""" + en: """Max batch size""" + zh: """最大批量请求大小""" } } @@ -163,7 +163,7 @@ When disabled the messages are buffered in RAM only.""" zh: """在较低消息率情况下尝试累积批量输出时的最大等待间隔，以提高资源的利用率。""" } label { - en: """Max Batch Wait Time""" + en: """Max batch wait time""" zh: """批量等待最大间隔""" } } From abf0329b60a66a6bb77dfd799354731501071f37 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 31 Mar 2023 13:03:00 +0200 Subject: [PATCH 16/50] test(emqx_banned_SUITE): fix flaky test case --- apps/emqx/test/emqx_banned_SUITE.erl | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 605c1de6d..80427ac47 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -154,7 +154,12 @@ t_session_taken(_) -> {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 120}} ]), - {ok, _} = emqtt:connect(C), + case emqtt:connect(C) of + {ok, _} -> + ok; + {error, econnrefused} -> + throw(mqtt_listener_not_ready) + end, {ok, _, [0]} = emqtt:subscribe(C, Topic, []), C end, @@ -168,9 +173,21 @@ t_session_taken(_) -> lists:seq(1, MsgNum) ) end, - - C1 = Connect(), - ok = emqtt:disconnect(C1), + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + try + C = Connect(), + emqtt:disconnect(C), + true + catch + throw:mqtt_listener_not_ready -> + false + end + end, + 3000 + ), Publish(), From 5011486b187c881a96919dd88b11d943da03968e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 30 Mar 2023 14:44:04 -0300 Subject: [PATCH 17/50] fix(kafka_consumer): return better error messages when probing kafka consumer bridge Fixes https://emqx.atlassian.net/browse/EMQX-9422 --- apps/emqx_resource/src/emqx_resource.erl | 4 +- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 56 +++++++++++++++---- .../kafka/emqx_bridge_impl_kafka_producer.erl | 5 +- .../emqx_bridge_impl_kafka_producer_SUITE.erl | 5 +- 4 files changed, 49 insertions(+), 21 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index a2fd9804b..0ed459c01 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -359,10 +359,10 @@ call_start(MgrId, Mod, Config) -> try Mod:on_start(MgrId, Config) catch - throw:{error, Error} -> + throw:Error -> {error, Error}; Kind:Error:Stacktrace -> - {error, {Kind, Error, Stacktrace}} + {error, #{exception => Kind, reason => Error, stacktrace => Stacktrace}} end. -spec call_health_check(manager_id(), module(), resource_state()) -> diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index 44ea95f90..f4dc3456e 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -95,6 +95,11 @@ commit_fun => brod_group_subscriber_v2:commit_fun() }. +-define(CLIENT_DOWN_MESSAGE, + "Failed to start Kafka client. Please check the logs for errors and check" + " the connection parameters." +). + %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- @@ -152,11 +157,7 @@ on_start(InstanceId, Config) -> kafka_hosts => BootstrapHosts, reason => emqx_misc:redact(Reason) }), - throw( - {error, - "Failed to start Kafka client. Please check the logs for errors and check" - " the connection parameters"} - ) + throw(?CLIENT_DOWN_MESSAGE) end, start_consumer(Config, InstanceId, ClientID). @@ -177,7 +178,7 @@ on_get_status(_InstanceID, State) -> kafka_client_id := ClientID, kafka_topics := KafkaTopics } = State, - do_get_status(ClientID, KafkaTopics, SubscriberId). + do_get_status(State, ClientID, KafkaTopics, SubscriberId). %%------------------------------------------------------------------------------------- %% `brod_group_subscriber' API @@ -374,22 +375,41 @@ stop_client(ClientID) -> ), ok. -do_get_status(ClientID, [KafkaTopic | RestTopics], SubscriberId) -> +do_get_status(State, ClientID, [KafkaTopic | RestTopics], SubscriberId) -> case brod:get_partitions_count(ClientID, KafkaTopic) of {ok, NPartitions} -> - case do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions) of - connected -> do_get_status(ClientID, RestTopics, SubscriberId); + case do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) of + connected -> do_get_status(State, ClientID, RestTopics, SubscriberId); disconnected -> disconnected end; + {error, {client_down, Context}} -> + case infer_client_error(Context) of + auth_error -> + Message = "Authentication error. " ++ ?CLIENT_DOWN_MESSAGE, + {disconnected, State, Message}; + {auth_error, Message0} -> + Message = binary_to_list(Message0) ++ "; " ++ ?CLIENT_DOWN_MESSAGE, + {disconnected, State, Message}; + connection_refused -> + Message = "Connection refused. " ++ ?CLIENT_DOWN_MESSAGE, + {disconnected, State, Message}; + _ -> + {disconnected, State, ?CLIENT_DOWN_MESSAGE} + end; + {error, leader_not_available} -> + Message = + "Leader connection not available. Please check the Kafka topic used," + " the connection parameters and Kafka cluster health", + {disconnected, State, Message}; _ -> disconnected end; -do_get_status(_ClientID, _KafkaTopics = [], _SubscriberId) -> +do_get_status(_State, _ClientID, _KafkaTopics = [], _SubscriberId) -> connected. --spec do_get_status(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> +-spec do_get_status1(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> connected | disconnected. -do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions) -> +do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) -> Results = lists:map( fun(N) -> @@ -508,3 +528,15 @@ encode(Value, base64) -> to_bin(B) when is_binary(B) -> B; to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). + +infer_client_error(Error) -> + case Error of + [{_BrokerEndpoint, {econnrefused, _}} | _] -> + connection_refused; + [{_BrokerEndpoint, {{sasl_auth_error, Message}, _}} | _] when is_binary(Message) -> + {auth_error, Message}; + [{_BrokerEndpoint, {{sasl_auth_error, _}, _}} | _] -> + auth_error; + _ -> + undefined + end. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index 8f5988aaf..09713a431 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -115,9 +115,8 @@ on_start(InstId, Config) -> } ), throw( - {error, - "Failed to start Kafka client. Please check the logs for errors and check" - " the connection parameters"} + "Failed to start Kafka client. Please check the logs for errors and check" + " the connection parameters." ) end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 2fb6c6857..9e32f818d 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -416,10 +416,7 @@ t_failed_creation_then_fix(Config) -> Type, erlang:list_to_atom(Name), WrongConf ), WrongConfigAtom = WrongConfigAtom1#{bridge_name => Name}, - ?assertThrow( - {error, _}, - ?PRODUCER:on_start(ResourceId, WrongConfigAtom) - ), + ?assertThrow(Reason when is_list(Reason), ?PRODUCER:on_start(ResourceId, WrongConfigAtom)), %% before throwing, it should cleanup the client process. we %% retry because the supervisor might need some time to really %% remove it from its tree. From c8dca74b188f4f60f3e99a1a1e89b75eb67db8ad Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 30 Mar 2023 17:19:55 +0800 Subject: [PATCH 18/50] chore: bump hackney version --- lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index d49ce59c0..e754bd573 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,7 @@ {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, diff --git a/mix.exs b/mix.exs index 514c9139d..600218e52 100644 --- a/mix.exs +++ b/mix.exs @@ -93,7 +93,7 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, - {:hackney, github: "benoitc/hackney", tag: "1.18.1", override: true} + {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index b641077ea..50a8124be 100644 --- a/rebar.config +++ b/rebar.config @@ -80,7 +80,7 @@ , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} - , {hackney, {git, "https://github.com/benoitc/hackney", {tag, "1.18.1"}}} + , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} ]}. {xref_ignores, From 36000abf51b284e5b6271d516878604d0ec956fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 16:50:21 +0200 Subject: [PATCH 19/50] refactor: relocate i18n files for apps/emqx --- .../emqx_authn_api_i18n.conf => rel/i18n/emqx_authn_api.hocon | 0 .../emqx_authn_http_i18n.conf => rel/i18n/emqx_authn_http.hocon | 0 .../emqx_authn_jwt_i18n.conf => rel/i18n/emqx_authn_jwt.hocon | 0 .../i18n/emqx_authn_mnesia.hocon | 0 .../i18n/emqx_authn_mongodb.hocon | 0 .../i18n/emqx_authn_mysql.hocon | 0 .../i18n/emqx_authn_pgsql.hocon | 0 .../i18n/emqx_authn_redis.hocon | 0 .../i18n/emqx_authn_schema.hocon | 0 .../i18n/emqx_authn_user_import_api.hocon | 0 .../i18n/emqx_authz_api_cache.hocon | 0 .../i18n/emqx_authz_api_mnesia.hocon | 0 .../i18n/emqx_authz_api_schema.hocon | 0 .../i18n/emqx_authz_api_settings.hocon | 0 .../i18n/emqx_authz_api_sources.hocon | 0 .../i18n/emqx_authz_schema.hocon | 0 .../i18n/emqx_auto_subscribe_api.hocon | 0 .../i18n/emqx_auto_subscribe_schema.hocon | 0 .../i18n/emqx_bridge_api.conf => rel/i18n/emqx_bridge_api.hocon | 0 .../i18n/emqx_bridge_mqtt_schema.hocon | 0 .../emqx_bridge_schema.conf => rel/i18n/emqx_bridge_schema.hocon | 0 .../i18n/emqx_bridge_webhook_schema.hocon | 0 .../i18n/emqx_coap_api_i18n.conf => rel/i18n/emqx_coap_api.hocon | 0 .../emqx_conf_schema.conf => rel/i18n/emqx_conf_schema.hocon | 0 .../emqx_connector_api.conf => rel/i18n/emqx_connector_api.hocon | 0 .../i18n/emqx_connector_http.hocon | 0 .../i18n/emqx_connector_ldap.hocon | 0 .../i18n/emqx_connector_mongo.hocon | 0 .../i18n/emqx_connector_mqtt.hocon | 0 .../i18n/emqx_connector_mqtt_schema.hocon | 0 .../i18n/emqx_connector_mysql.hocon | 0 .../i18n/emqx_connector_pgsql.hocon | 0 .../i18n/emqx_connector_redis.hocon | 0 .../i18n/emqx_connector_schema_lib.hocon | 0 .../i18n/emqx_dashboard_api.hocon | 0 .../i18n/emqx_dashboard_schema.hocon | 0 .../i18n/emqx_delayed_api.hocon | 0 .../i18n/emqx_ee_bridge_cassa.hocon | 0 .../i18n/emqx_ee_bridge_clickhouse.hocon | 0 .../i18n/emqx_ee_bridge_dynamo.hocon | 0 .../i18n/emqx_ee_bridge_gcp_pubsub.hocon | 0 .../i18n/emqx_ee_bridge_hstreamdb.hocon | 0 .../i18n/emqx_ee_bridge_influxdb.hocon | 0 .../i18n/emqx_ee_bridge_kafka.hocon | 0 .../i18n/emqx_ee_bridge_mongodb.hocon | 0 .../i18n/emqx_ee_bridge_mysql.hocon | 0 .../i18n/emqx_ee_bridge_pgsql.hocon | 0 .../i18n/emqx_ee_bridge_redis.hocon | 0 .../i18n/emqx_ee_bridge_rocketmq.hocon | 0 .../i18n/emqx_ee_bridge_tdengine.hocon | 0 .../i18n/emqx_ee_connector_cassa.hocon | 0 .../i18n/emqx_ee_connector_clickhouse.hocon | 1 - .../i18n/emqx_ee_connector_dynamo.hocon | 0 .../i18n/emqx_ee_connector_hstreamdb.hocon | 0 .../i18n/emqx_ee_connector_influxdb.hocon | 0 .../i18n/emqx_ee_connector_rocketmq.hocon | 0 .../i18n/emqx_ee_connector_tdengine.hocon | 0 .../emqx_exhook_api_i18n.conf => rel/i18n/emqx_exhook_api.hocon | 0 .../emqx_exhook_i18n.conf => rel/i18n/emqx_exhook_schema.hocon | 0 .../i18n/emqx_gateway_api.hocon | 0 .../i18n/emqx_gateway_api_authn.hocon | 0 .../i18n/emqx_gateway_api_clients.hocon | 0 .../i18n/emqx_gateway_api_listeners.hocon | 0 .../i18n/emqx_gateway_schema.hocon | 0 .../i18n/emqx_license_http_api.hocon | 0 .../i18n/emqx_license_schema.hocon | 0 .../emqx_limiter_i18n.conf => rel/i18n/emqx_limiter_schema.hocon | 0 .../emqx_lwm2m_api_i18n.conf => rel/i18n/emqx_lwm2m_api.hocon | 0 .../i18n/emqx_mgmt_api_alarms.hocon | 0 .../i18n/emqx_mgmt_api_banned.hocon | 0 .../i18n/emqx_mgmt_api_key_schema.hocon | 0 .../i18n/emqx_mgmt_api_publish.hocon | 1 - .../i18n/emqx_mgmt_api_status.hocon | 0 .../i18n/emqx_modules_schema.hocon | 0 .../i18n/emqx_plugins_schema.hocon | 0 .../i18n/emqx_prometheus_schema.hocon | 0 .../i18n/emqx_psk_i18n.conf => rel/i18n/emqx_psk_schema.hocon | 0 .../i18n/emqx_resource_schema.hocon | 0 .../i18n/emqx_retainer_api.hocon | 0 .../i18n/emqx_retainer_schema.hocon | 0 .../i18n/emqx_rewrite_api.hocon | 0 .../i18n/emqx_rule_api_schema.hocon | 0 .../i18n/emqx_rule_engine_api.hocon | 0 .../i18n/emqx_rule_engine_schema.hocon | 0 .../i18n/emqx_schema_i18n.conf => rel/i18n/emqx_schema.hocon | 0 .../i18n/emqx_slow_subs_api.hocon | 0 .../i18n/emqx_slow_subs_schema.hocon | 0 .../emqx_statsd_api_i18n.conf => rel/i18n/emqx_statsd_api.hocon | 0 .../i18n/emqx_statsd_schema.hocon | 0 .../i18n/emqx_telemetry_api.hocon | 0 .../i18n/emqx_topic_metrics_api.hocon | 0 91 files changed, 2 deletions(-) rename apps/emqx_authn/i18n/emqx_authn_api_i18n.conf => rel/i18n/emqx_authn_api.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_http_i18n.conf => rel/i18n/emqx_authn_http.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf => rel/i18n/emqx_authn_jwt.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf => rel/i18n/emqx_authn_mnesia.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf => rel/i18n/emqx_authn_mongodb.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf => rel/i18n/emqx_authn_mysql.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf => rel/i18n/emqx_authn_pgsql.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf => rel/i18n/emqx_authn_redis.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf => rel/i18n/emqx_authn_schema.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf => rel/i18n/emqx_authn_user_import_api.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf => rel/i18n/emqx_authz_api_cache.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf => rel/i18n/emqx_authz_api_mnesia.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf => rel/i18n/emqx_authz_api_schema.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf => rel/i18n/emqx_authz_api_settings.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf => rel/i18n/emqx_authz_api_sources.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf => rel/i18n/emqx_authz_schema.hocon (100%) rename apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf => rel/i18n/emqx_auto_subscribe_api.hocon (100%) rename apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf => rel/i18n/emqx_auto_subscribe_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_api.conf => rel/i18n/emqx_bridge_api.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf => rel/i18n/emqx_bridge_mqtt_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_schema.conf => rel/i18n/emqx_bridge_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf => rel/i18n/emqx_bridge_webhook_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf => rel/i18n/emqx_coap_api.hocon (100%) rename apps/emqx_conf/i18n/emqx_conf_schema.conf => rel/i18n/emqx_conf_schema.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_api.conf => rel/i18n/emqx_connector_api.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_http.conf => rel/i18n/emqx_connector_http.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_ldap.conf => rel/i18n/emqx_connector_ldap.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mongo.conf => rel/i18n/emqx_connector_mongo.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mqtt.conf => rel/i18n/emqx_connector_mqtt.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf => rel/i18n/emqx_connector_mqtt_schema.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mysql.conf => rel/i18n/emqx_connector_mysql.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_pgsql.conf => rel/i18n/emqx_connector_pgsql.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_redis.conf => rel/i18n/emqx_connector_redis.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_schema_lib.conf => rel/i18n/emqx_connector_schema_lib.hocon (100%) rename apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf => rel/i18n/emqx_dashboard_api.hocon (100%) rename apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf => rel/i18n/emqx_dashboard_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf => rel/i18n/emqx_delayed_api.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf => rel/i18n/emqx_ee_bridge_cassa.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf => rel/i18n/emqx_ee_bridge_clickhouse.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf => rel/i18n/emqx_ee_bridge_dynamo.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf => rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf => rel/i18n/emqx_ee_bridge_hstreamdb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf => rel/i18n/emqx_ee_bridge_influxdb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf => rel/i18n/emqx_ee_bridge_kafka.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf => rel/i18n/emqx_ee_bridge_mongodb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf => rel/i18n/emqx_ee_bridge_mysql.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf => rel/i18n/emqx_ee_bridge_pgsql.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf => rel/i18n/emqx_ee_bridge_redis.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf => rel/i18n/emqx_ee_bridge_rocketmq.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf => rel/i18n/emqx_ee_bridge_tdengine.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf => rel/i18n/emqx_ee_connector_cassa.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf => rel/i18n/emqx_ee_connector_clickhouse.hocon (99%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf => rel/i18n/emqx_ee_connector_dynamo.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf => rel/i18n/emqx_ee_connector_hstreamdb.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf => rel/i18n/emqx_ee_connector_influxdb.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf => rel/i18n/emqx_ee_connector_rocketmq.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf => rel/i18n/emqx_ee_connector_tdengine.hocon (100%) rename apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf => rel/i18n/emqx_exhook_api.hocon (100%) rename apps/emqx_exhook/i18n/emqx_exhook_i18n.conf => rel/i18n/emqx_exhook_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf => rel/i18n/emqx_gateway_api.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf => rel/i18n/emqx_gateway_api_authn.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf => rel/i18n/emqx_gateway_api_clients.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf => rel/i18n/emqx_gateway_api_listeners.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf => rel/i18n/emqx_gateway_schema.hocon (100%) rename lib-ee/emqx_license/i18n/emqx_license_http_api.conf => rel/i18n/emqx_license_http_api.hocon (100%) rename lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf => rel/i18n/emqx_license_schema.hocon (100%) rename apps/emqx/i18n/emqx_limiter_i18n.conf => rel/i18n/emqx_limiter_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf => rel/i18n/emqx_lwm2m_api.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf => rel/i18n/emqx_mgmt_api_alarms.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf => rel/i18n/emqx_mgmt_api_banned.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf => rel/i18n/emqx_mgmt_api_key_schema.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf => rel/i18n/emqx_mgmt_api_publish.hocon (99%) rename apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf => rel/i18n/emqx_mgmt_api_status.hocon (100%) rename apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf => rel/i18n/emqx_modules_schema.hocon (100%) rename apps/emqx_plugins/i18n/emqx_plugins_schema.conf => rel/i18n/emqx_plugins_schema.hocon (100%) rename apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf => rel/i18n/emqx_prometheus_schema.hocon (100%) rename apps/emqx_psk/i18n/emqx_psk_i18n.conf => rel/i18n/emqx_psk_schema.hocon (100%) rename apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf => rel/i18n/emqx_resource_schema.hocon (100%) rename apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf => rel/i18n/emqx_retainer_api.hocon (100%) rename apps/emqx_retainer/i18n/emqx_retainer_i18n.conf => rel/i18n/emqx_retainer_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf => rel/i18n/emqx_rewrite_api.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf => rel/i18n/emqx_rule_api_schema.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf => rel/i18n/emqx_rule_engine_api.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf => rel/i18n/emqx_rule_engine_schema.hocon (100%) rename apps/emqx/i18n/emqx_schema_i18n.conf => rel/i18n/emqx_schema.hocon (100%) rename apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf => rel/i18n/emqx_slow_subs_api.hocon (100%) rename apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf => rel/i18n/emqx_slow_subs_schema.hocon (100%) rename apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf => rel/i18n/emqx_statsd_api.hocon (100%) rename apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf => rel/i18n/emqx_statsd_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf => rel/i18n/emqx_telemetry_api.hocon (100%) rename apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf => rel/i18n/emqx_topic_metrics_api.hocon (100%) diff --git a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf b/rel/i18n/emqx_authn_api.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_api_i18n.conf rename to rel/i18n/emqx_authn_api.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_http_i18n.conf b/rel/i18n/emqx_authn_http.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_http_i18n.conf rename to rel/i18n/emqx_authn_http.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf b/rel/i18n/emqx_authn_jwt.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf rename to rel/i18n/emqx_authn_jwt.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf b/rel/i18n/emqx_authn_mnesia.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf rename to rel/i18n/emqx_authn_mnesia.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf b/rel/i18n/emqx_authn_mongodb.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf rename to rel/i18n/emqx_authn_mongodb.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf b/rel/i18n/emqx_authn_mysql.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf rename to rel/i18n/emqx_authn_mysql.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf b/rel/i18n/emqx_authn_pgsql.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf rename to rel/i18n/emqx_authn_pgsql.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf b/rel/i18n/emqx_authn_redis.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf rename to rel/i18n/emqx_authn_redis.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf b/rel/i18n/emqx_authn_schema.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf rename to rel/i18n/emqx_authn_schema.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf b/rel/i18n/emqx_authn_user_import_api.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf rename to rel/i18n/emqx_authn_user_import_api.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf b/rel/i18n/emqx_authz_api_cache.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf rename to rel/i18n/emqx_authz_api_cache.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf b/rel/i18n/emqx_authz_api_mnesia.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf rename to rel/i18n/emqx_authz_api_mnesia.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf b/rel/i18n/emqx_authz_api_schema.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf rename to rel/i18n/emqx_authz_api_schema.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf b/rel/i18n/emqx_authz_api_settings.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf rename to rel/i18n/emqx_authz_api_settings.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf b/rel/i18n/emqx_authz_api_sources.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf rename to rel/i18n/emqx_authz_api_sources.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf b/rel/i18n/emqx_authz_schema.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf rename to rel/i18n/emqx_authz_schema.hocon diff --git a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf b/rel/i18n/emqx_auto_subscribe_api.hocon similarity index 100% rename from apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf rename to rel/i18n/emqx_auto_subscribe_api.hocon diff --git a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf b/rel/i18n/emqx_auto_subscribe_schema.hocon similarity index 100% rename from apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf rename to rel/i18n/emqx_auto_subscribe_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_api.conf b/rel/i18n/emqx_bridge_api.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_api.conf rename to rel/i18n/emqx_bridge_api.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf b/rel/i18n/emqx_bridge_mqtt_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf rename to rel/i18n/emqx_bridge_mqtt_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/rel/i18n/emqx_bridge_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_schema.conf rename to rel/i18n/emqx_bridge_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/rel/i18n/emqx_bridge_webhook_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf rename to rel/i18n/emqx_bridge_webhook_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf b/rel/i18n/emqx_coap_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf rename to rel/i18n/emqx_coap_api.hocon diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/rel/i18n/emqx_conf_schema.hocon similarity index 100% rename from apps/emqx_conf/i18n/emqx_conf_schema.conf rename to rel/i18n/emqx_conf_schema.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_api.conf b/rel/i18n/emqx_connector_api.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_api.conf rename to rel/i18n/emqx_connector_api.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_http.conf b/rel/i18n/emqx_connector_http.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_http.conf rename to rel/i18n/emqx_connector_http.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_ldap.conf b/rel/i18n/emqx_connector_ldap.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_ldap.conf rename to rel/i18n/emqx_connector_ldap.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mongo.conf b/rel/i18n/emqx_connector_mongo.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mongo.conf rename to rel/i18n/emqx_connector_mongo.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt.conf b/rel/i18n/emqx_connector_mqtt.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mqtt.conf rename to rel/i18n/emqx_connector_mqtt.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/rel/i18n/emqx_connector_mqtt_schema.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf rename to rel/i18n/emqx_connector_mqtt_schema.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mysql.conf b/rel/i18n/emqx_connector_mysql.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mysql.conf rename to rel/i18n/emqx_connector_mysql.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_pgsql.conf b/rel/i18n/emqx_connector_pgsql.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_pgsql.conf rename to rel/i18n/emqx_connector_pgsql.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/rel/i18n/emqx_connector_redis.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_redis.conf rename to rel/i18n/emqx_connector_redis.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf b/rel/i18n/emqx_connector_schema_lib.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_schema_lib.conf rename to rel/i18n/emqx_connector_schema_lib.hocon diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf b/rel/i18n/emqx_dashboard_api.hocon similarity index 100% rename from apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf rename to rel/i18n/emqx_dashboard_api.hocon diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/rel/i18n/emqx_dashboard_schema.hocon similarity index 100% rename from apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf rename to rel/i18n/emqx_dashboard_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf b/rel/i18n/emqx_delayed_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf rename to rel/i18n/emqx_delayed_api.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf b/rel/i18n/emqx_ee_bridge_cassa.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf rename to rel/i18n/emqx_ee_bridge_cassa.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf b/rel/i18n/emqx_ee_bridge_clickhouse.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf rename to rel/i18n/emqx_ee_bridge_clickhouse.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf b/rel/i18n/emqx_ee_bridge_dynamo.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf rename to rel/i18n/emqx_ee_bridge_dynamo.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf b/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf rename to rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf rename to rel/i18n/emqx_ee_bridge_hstreamdb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/rel/i18n/emqx_ee_bridge_influxdb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf rename to rel/i18n/emqx_ee_bridge_influxdb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/rel/i18n/emqx_ee_bridge_kafka.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf rename to rel/i18n/emqx_ee_bridge_kafka.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/rel/i18n/emqx_ee_bridge_mongodb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf rename to rel/i18n/emqx_ee_bridge_mongodb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/rel/i18n/emqx_ee_bridge_mysql.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf rename to rel/i18n/emqx_ee_bridge_mysql.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf b/rel/i18n/emqx_ee_bridge_pgsql.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf rename to rel/i18n/emqx_ee_bridge_pgsql.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf b/rel/i18n/emqx_ee_bridge_redis.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf rename to rel/i18n/emqx_ee_bridge_redis.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf b/rel/i18n/emqx_ee_bridge_rocketmq.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf rename to rel/i18n/emqx_ee_bridge_rocketmq.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf b/rel/i18n/emqx_ee_bridge_tdengine.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf rename to rel/i18n/emqx_ee_bridge_tdengine.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf b/rel/i18n/emqx_ee_connector_cassa.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf rename to rel/i18n/emqx_ee_connector_cassa.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf b/rel/i18n/emqx_ee_connector_clickhouse.hocon similarity index 99% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf rename to rel/i18n/emqx_ee_connector_clickhouse.hocon index 069505a69..4d30e1715 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf +++ b/rel/i18n/emqx_ee_connector_clickhouse.hocon @@ -1,4 +1,3 @@ - emqx_ee_connector_clickhouse { base_url { diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf b/rel/i18n/emqx_ee_connector_dynamo.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf rename to rel/i18n/emqx_ee_connector_dynamo.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf b/rel/i18n/emqx_ee_connector_hstreamdb.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf rename to rel/i18n/emqx_ee_connector_hstreamdb.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/rel/i18n/emqx_ee_connector_influxdb.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf rename to rel/i18n/emqx_ee_connector_influxdb.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf b/rel/i18n/emqx_ee_connector_rocketmq.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf rename to rel/i18n/emqx_ee_connector_rocketmq.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf b/rel/i18n/emqx_ee_connector_tdengine.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf rename to rel/i18n/emqx_ee_connector_tdengine.hocon diff --git a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf b/rel/i18n/emqx_exhook_api.hocon similarity index 100% rename from apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf rename to rel/i18n/emqx_exhook_api.hocon diff --git a/apps/emqx_exhook/i18n/emqx_exhook_i18n.conf b/rel/i18n/emqx_exhook_schema.hocon similarity index 100% rename from apps/emqx_exhook/i18n/emqx_exhook_i18n.conf rename to rel/i18n/emqx_exhook_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/rel/i18n/emqx_gateway_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf rename to rel/i18n/emqx_gateway_api.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/rel/i18n/emqx_gateway_api_authn.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf rename to rel/i18n/emqx_gateway_api_authn.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf b/rel/i18n/emqx_gateway_api_clients.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf rename to rel/i18n/emqx_gateway_api_clients.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/rel/i18n/emqx_gateway_api_listeners.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf rename to rel/i18n/emqx_gateway_api_listeners.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/rel/i18n/emqx_gateway_schema.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf rename to rel/i18n/emqx_gateway_schema.hocon diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/rel/i18n/emqx_license_http_api.hocon similarity index 100% rename from lib-ee/emqx_license/i18n/emqx_license_http_api.conf rename to rel/i18n/emqx_license_http_api.hocon diff --git a/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf b/rel/i18n/emqx_license_schema.hocon similarity index 100% rename from lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf rename to rel/i18n/emqx_license_schema.hocon diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/rel/i18n/emqx_limiter_schema.hocon similarity index 100% rename from apps/emqx/i18n/emqx_limiter_i18n.conf rename to rel/i18n/emqx_limiter_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf b/rel/i18n/emqx_lwm2m_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf rename to rel/i18n/emqx_lwm2m_api.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/rel/i18n/emqx_mgmt_api_alarms.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf rename to rel/i18n/emqx_mgmt_api_alarms.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/rel/i18n/emqx_mgmt_api_banned.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf rename to rel/i18n/emqx_mgmt_api_banned.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf b/rel/i18n/emqx_mgmt_api_key_schema.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf rename to rel/i18n/emqx_mgmt_api_key_schema.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/rel/i18n/emqx_mgmt_api_publish.hocon similarity index 99% rename from apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf rename to rel/i18n/emqx_mgmt_api_publish.hocon index f91115df5..a09732cfc 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf +++ b/rel/i18n/emqx_mgmt_api_publish.hocon @@ -1,4 +1,3 @@ - emqx_mgmt_api_publish { publish_api { desc { diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/rel/i18n/emqx_mgmt_api_status.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf rename to rel/i18n/emqx_mgmt_api_status.hocon diff --git a/apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf b/rel/i18n/emqx_modules_schema.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf rename to rel/i18n/emqx_modules_schema.hocon diff --git a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf b/rel/i18n/emqx_plugins_schema.hocon similarity index 100% rename from apps/emqx_plugins/i18n/emqx_plugins_schema.conf rename to rel/i18n/emqx_plugins_schema.hocon diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/rel/i18n/emqx_prometheus_schema.hocon similarity index 100% rename from apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf rename to rel/i18n/emqx_prometheus_schema.hocon diff --git a/apps/emqx_psk/i18n/emqx_psk_i18n.conf b/rel/i18n/emqx_psk_schema.hocon similarity index 100% rename from apps/emqx_psk/i18n/emqx_psk_i18n.conf rename to rel/i18n/emqx_psk_schema.hocon diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/rel/i18n/emqx_resource_schema.hocon similarity index 100% rename from apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf rename to rel/i18n/emqx_resource_schema.hocon diff --git a/apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf b/rel/i18n/emqx_retainer_api.hocon similarity index 100% rename from apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf rename to rel/i18n/emqx_retainer_api.hocon diff --git a/apps/emqx_retainer/i18n/emqx_retainer_i18n.conf b/rel/i18n/emqx_retainer_schema.hocon similarity index 100% rename from apps/emqx_retainer/i18n/emqx_retainer_i18n.conf rename to rel/i18n/emqx_retainer_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf b/rel/i18n/emqx_rewrite_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf rename to rel/i18n/emqx_rewrite_api.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf b/rel/i18n/emqx_rule_api_schema.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf rename to rel/i18n/emqx_rule_api_schema.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf b/rel/i18n/emqx_rule_engine_api.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf rename to rel/i18n/emqx_rule_engine_api.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf b/rel/i18n/emqx_rule_engine_schema.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf rename to rel/i18n/emqx_rule_engine_schema.hocon diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/rel/i18n/emqx_schema.hocon similarity index 100% rename from apps/emqx/i18n/emqx_schema_i18n.conf rename to rel/i18n/emqx_schema.hocon diff --git a/apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf b/rel/i18n/emqx_slow_subs_api.hocon similarity index 100% rename from apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf rename to rel/i18n/emqx_slow_subs_api.hocon diff --git a/apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf b/rel/i18n/emqx_slow_subs_schema.hocon similarity index 100% rename from apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf rename to rel/i18n/emqx_slow_subs_schema.hocon diff --git a/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf b/rel/i18n/emqx_statsd_api.hocon similarity index 100% rename from apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf rename to rel/i18n/emqx_statsd_api.hocon diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/rel/i18n/emqx_statsd_schema.hocon similarity index 100% rename from apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf rename to rel/i18n/emqx_statsd_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf b/rel/i18n/emqx_telemetry_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf rename to rel/i18n/emqx_telemetry_api.hocon diff --git a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf b/rel/i18n/emqx_topic_metrics_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf rename to rel/i18n/emqx_topic_metrics_api.hocon From 7ec9b9a40888f3a7f2cb61c867684b21745b0b07 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 18:20:02 +0200 Subject: [PATCH 20/50] refactor(merge-i18n.escript): merge files in rel/i18n --- scripts/merge-i18n.escript | 48 +++++--------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 816cbe182..b2501d10a 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -4,12 +4,8 @@ main(_) -> BaseConf = <<"">>, - Cfgs0 = get_all_cfgs("apps/"), - Cfgs1 = get_all_cfgs("lib-ee/"), - Conf0 = merge(BaseConf, Cfgs0), - Conf = [merge(Conf0, Cfgs1), - io_lib:nl() - ], + Cfgs0 = get_all_files(), + Conf = merge(BaseConf, Cfgs0), OutputFile = "apps/emqx_dashboard/priv/i18n.conf", ok = filelib:ensure_dir(OutputFile), ok = file:write_file(OutputFile, Conf). @@ -25,39 +21,7 @@ merge(BaseConf, Cfgs) -> end end, BaseConf, Cfgs). -get_all_cfgs(Root) -> - Apps = filelib:wildcard("*", Root) -- ["emqx_machine"], - Dirs = [filename:join([Root, App]) || App <- Apps], - lists:foldl(fun get_cfgs/2, [], Dirs). - -get_all_cfgs(Dir, Cfgs) -> - Fun = fun(E, Acc) -> - Path = filename:join([Dir, E]), - get_cfgs(Path, Acc) - end, - lists:foldl(Fun, Cfgs, filelib:wildcard("*", Dir)). - -get_cfgs(Dir, Cfgs) -> - case filelib:is_dir(Dir) of - false -> - Cfgs; - _ -> - Files = filelib:wildcard("*", Dir), - case lists:member("i18n", Files) of - false -> - try_enter_child(Dir, Files, Cfgs); - true -> - EtcDir = filename:join([Dir, "i18n"]), - Confs = filelib:wildcard("*.conf", EtcDir), - NewCfgs = [filename:join([EtcDir, Name]) || Name <- Confs], - try_enter_child(Dir, Files, NewCfgs ++ Cfgs) - end - end. - -try_enter_child(Dir, Files, Cfgs) -> - case lists:member("src", Files) of - false -> - Cfgs; - true -> - get_all_cfgs(filename:join([Dir, "src"]), Cfgs) - end. +get_all_files() -> + Dir = filename:join(["rel","i18n"]), + Files = filelib:wildcard("*.hocon", Dir), + lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). From c17de6c415215234cdc98c58b3986b91827a7269 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 21:42:03 +0200 Subject: [PATCH 21/50] docs: fix i18n desc style --- rel/i18n/emqx_ee_bridge_clickhouse.hocon | 6 ++---- rel/i18n/emqx_ee_bridge_kafka.hocon | 4 ++-- rel/i18n/emqx_ee_bridge_tdengine.hocon | 6 ++---- rel/i18n/emqx_ee_connector_dynamo.hocon | 4 ++-- rel/i18n/emqx_ee_connector_rocketmq.hocon | 12 ++++-------- rel/i18n/emqx_ee_connector_tdengine.hocon | 12 ++++-------- rel/i18n/emqx_schema.hocon | 4 ++-- scripts/check-i18n-style.escript | 2 ++ scripts/check-i18n-style.sh | 2 +- 9 files changed, 21 insertions(+), 31 deletions(-) diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_ee_bridge_clickhouse.hocon index 5096f8590..5bc6f12b2 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_ee_bridge_clickhouse.hocon @@ -6,11 +6,9 @@ emqx_ee_bridge_clickhouse { matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded. -""" +will be forwarded.""" zh: """发送到 'local_topic' 的消息都会转发到 Clickhouse。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 -""" +注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" } label { en: "Local Topic" diff --git a/rel/i18n/emqx_ee_bridge_kafka.hocon b/rel/i18n/emqx_ee_bridge_kafka.hocon index df32c1cae..d1a017416 100644 --- a/rel/i18n/emqx_ee_bridge_kafka.hocon +++ b/rel/i18n/emqx_ee_bridge_kafka.hocon @@ -547,7 +547,7 @@ emqx_ee_bridge_kafka { "ts: message timestamp.\n" "ts_type: message timestamp type, which is one of" " create, append or undefined.\n" - "value: Kafka message value (uses the chosen value encoding).\n" + "value: Kafka message value (uses the chosen value encoding)." zh: "用于转换收到的 Kafka 消息的模板。 " "默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 " "这些字段包括：" @@ -558,7 +558,7 @@ emqx_ee_bridge_kafka { "ts: 消息的时间戳。\n" "ts_type：消息的时间戳类型，值可能是：" " create， append 或 undefined。\n" - "value: Kafka 消息值（使用选择的编码方式编码）。\n" + "value: Kafka 消息值（使用选择的编码方式编码）。" } label { diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_ee_bridge_tdengine.hocon index 2d5af9f16..21fc013df 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_ee_bridge_tdengine.hocon @@ -6,11 +6,9 @@ emqx_ee_bridge_tdengine { matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded. -""" +will be forwarded.""" zh: """发送到 'local_topic' 的消息都会转发到 TDengine。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 -""" +注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" } label { en: "Local Topic" diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index e1fc11e03..295929a72 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -2,8 +2,8 @@ emqx_ee_connector_dynamo { url { desc { - en: """The url of DynamoDB endpoint.
""" - zh: """DynamoDB 的地址。
""" + en: """The url of DynamoDB endpoint.""" + zh: """DynamoDB 的地址。""" } label: { en: "DynamoDB Endpoint" diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index d4a610212..44dda7931 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -2,16 +2,12 @@ emqx_ee_connector_rocketmq { server { desc { - en: """ -The IPv4 or IPv6 address or the hostname to connect to.
+ en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
-The RocketMQ default port 9876 is used if `[:Port]` is not specified. -""" - zh: """ -将要连接的 IPv4 或 IPv6 地址，或者主机名。
+The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。 -""" +如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" } label: { en: "Server Host" diff --git a/rel/i18n/emqx_ee_connector_tdengine.hocon b/rel/i18n/emqx_ee_connector_tdengine.hocon index c6c58d82d..02254124c 100644 --- a/rel/i18n/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/emqx_ee_connector_tdengine.hocon @@ -2,16 +2,12 @@ emqx_ee_connector_tdengine { server { desc { - en: """ -The IPv4 or IPv6 address or the hostname to connect to.
+ en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
-The TDengine default port 6041 is used if `[:Port]` is not specified. -""" - zh: """ -将要连接的 IPv4 或 IPv6 地址，或者主机名。
+The TDengine default port 6041 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。 -""" +如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" } label: { en: "Server Host" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 28c58713d..d36809c3b 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1505,8 +1505,8 @@ In case PSK cipher suites are intended, make sure to configure common_ssl_opts_schema_hibernate_after { desc { - en: """ Hibernate the SSL process after idling for amount of time reducing its memory footprint. """ - zh: """ 在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" + en: """Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" + zh: """在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" } label: { en: "hibernate after" diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index 6ad6c1770..7e90f0807 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -1,5 +1,7 @@ #!/usr/bin/env escript +%% called from check-i18n-style.sh + -mode(compile). -define(YELLOW, "\e[33m"). diff --git a/scripts/check-i18n-style.sh b/scripts/check-i18n-style.sh index 0be565f30..d21f43a72 100755 --- a/scripts/check-i18n-style.sh +++ b/scripts/check-i18n-style.sh @@ -3,6 +3,6 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")/.." -all_files="$(git ls-files '*i18n*.conf')" +all_files="$(git ls-files 'rel/i18n/*.hocon')" ./scripts/check-i18n-style.escript "$all_files" From bdd3960e178319698a8a45b6da404e405eada777 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 21:48:27 +0200 Subject: [PATCH 22/50] build: check i18n style earlier --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13e3a6d43..a5adf0e0a 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ ct: $(REBAR) merge-config static_checks: @$(REBAR) as check do xref, dialyzer @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then $(REBAR) ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE); fi - @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then ./scripts/check-i18n-style.sh; fi + ./scripts/check-i18n-style.sh APPS=$(shell $(SCRIPTS)/find-apps.sh) From 5f6d318cf0cadeb6048dec407adffac32b2d8419 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 23:14:01 +0200 Subject: [PATCH 23/50] fix(i18n): fix missing docs for gateway configs --- apps/emqx_gateway/src/emqx_gateway_schema.erl | 34 +++++++++--------- rel/i18n/emqx_gateway_schema.hocon | 36 +++---------------- rel/i18n/emqx_rule_api_schema.hocon | 4 +-- scripts/check-i18n-style.escript | 7 ++-- 4 files changed, 27 insertions(+), 54 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 2034a40eb..741fb98ae 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -453,20 +453,20 @@ fields(translator) -> ]; fields(udp_listeners) -> [ - {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, - {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(dtls_listener)})} + {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_listeners) -> [ - {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(tcp_listener)})}, - {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(ssl_listener)})} + {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_udp_listeners) -> [ - {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(tcp_listener)})}, - {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(ssl_listener)})}, - {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, - {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(dtls_listener)})} + {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_listener) -> %% some special configs for tcp listener @@ -558,19 +558,19 @@ desc(udp_listeners) -> desc(tcp_listeners) -> "Settings for the TCP listeners."; desc(tcp_udp_listeners) -> - "Settings for the listeners."; + "Settings for TCP and UDP listeners."; desc(tcp_listener) -> - "Settings for the TCP listener."; + "Settings for TCP listener."; desc(ssl_listener) -> - "Settings for the SSL listener."; + "Settings for SSL listener."; desc(udp_listener) -> - "Settings for the UDP listener."; + "Settings for UDP listener."; desc(dtls_listener) -> - "Settings for the DTLS listener."; + "Settings for DTLS listener."; desc(udp_opts) -> - "Settings for the UDP sockets."; + "Settings for UDP sockets."; desc(dtls_opts) -> - "Settings for the DTLS protocol."; + "Settings for DTLS protocol."; desc(_) -> undefined. @@ -625,7 +625,7 @@ mountpoint(Default) -> binary(), #{ default => iolist_to_binary(Default), - desc => ?DESC(gateway_common_mountpoint) + desc => ?DESC(gateway_mountpoint) } ). @@ -674,7 +674,7 @@ common_listener_opts() -> binary(), #{ default => undefined, - desc => ?DESC(gateway_common_listener_mountpoint) + desc => ?DESC(gateway_mountpoint) } )}, {access_rules, diff --git a/rel/i18n/emqx_gateway_schema.hocon b/rel/i18n/emqx_gateway_schema.hocon index 74a70eb73..ebc955557 100644 --- a/rel/i18n/emqx_gateway_schema.hocon +++ b/rel/i18n/emqx_gateway_schema.hocon @@ -370,13 +370,6 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e } } - gateway_common_mountpoint { - desc { - en: """""" - zh: """""" - } - } - gateway_common_clientinfo_override { desc { en: """ClientInfo override.""" @@ -431,10 +424,10 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e } } - tcp_listener { + listener_name_to_settings_map{ desc { - en: """""" - zh: """""" + en: """A map from listener names to listener settings.""" + zh: """从监听器名称到配置参数的映射。""" } } @@ -468,13 +461,6 @@ EMQX will close the TCP connection if proxy protocol packet is not received with } } - ssl_listener { - desc { - en: """""" - zh: """""" - } - } - ssl_listener_options { desc { en: """SSL Socket options.""" @@ -482,13 +468,6 @@ EMQX will close the TCP connection if proxy protocol packet is not received with } } - udp_listener { - desc { - en: """""" - zh: """""" - } - } - udp_listener_udp_opts { desc { en: """Settings for the UDP sockets.""" @@ -533,13 +512,6 @@ See: https://erlang.org/doc/man/inet.html#setopts-2""" } } - dtls_listener { - desc { - en: """""" - zh: """""" - } - } - dtls_listener_acceptors { desc { en: """Size of the acceptor pool.""" @@ -592,7 +564,7 @@ When set to false clients will be allowed to connect without authen } } - gateway_common_listener_mountpoint { + gateway_mountpoint { desc { en: """When publishing or subscribing, prefix all topics with a mountpoint string. The prefixed string will be removed from the topic name when the message is delivered to the subscriber. diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index e4c2314de..f9b344666 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -35,8 +35,8 @@ emqx_rule_api_schema { event_username { desc { - en: "The User Name" - zh: "" + en: "Username" + zh: "用户名" } label: { en: "Username" diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index 7e90f0807..cbe79c82e 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -4,11 +4,12 @@ -mode(compile). --define(YELLOW, "\e[33m"). +% -define(YELLOW, "\e[33m"). % not used -define(RED, "\e[31m"). -define(RESET, "\e[39m"). main([Files0]) -> + io:format(user, "checking i18n file styles", []), _ = put(errors, 0), Files = string:tokens(Files0, "\n"), ok = load_hocon(), @@ -48,7 +49,7 @@ logerr(Fmt, Args) -> check(File) -> - io:format(user, "checking: ~s~n", [File]), + io:format(user, ".", []), {ok, C} = hocon:load(File), maps:foreach(fun check_one_field/2, C), ok. @@ -86,7 +87,7 @@ do_check_desc(Name, _) -> die("~s: missing 'zh' or 'en'~n", [Name]). check_desc_string(Name, Tr, <<>>) -> - io:format(standard_error, ?YELLOW ++ "WARNING: ~s.~s: empty string~n" ++ ?RESET, [Name, Tr]); + logerr("~s.~s: empty string~n", [Name, Tr]); check_desc_string(Name, Tr, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> From 677b76afce1ac7826f5d2e4a3aab9ca332eaae00 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 13:13:43 +0200 Subject: [PATCH 24/50] refactor: replace hidden => true with importance => hidden --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index e11ef1c93..529e28dea 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -233,7 +233,7 @@ fields(socket_opts) -> boolean(), #{ default => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(socket_nodelay) } )} From 53712e61462e8e617783f2886c2a17387f460ae3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 30 Mar 2023 15:11:06 +0800 Subject: [PATCH 25/50] fix: running nodes should not include replica nodes --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_app.erl | 10 +++++----- changes/ce/fix-10313.en.md | 2 ++ changes/ce/fix-10313.zh.md | 2 ++ 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 changes/ce/fix-10313.en.md create mode 100644 changes/ce/fix-10313.zh.md diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index fbbffba1f..37707431a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 34224c3f2..af42f0e1a 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -90,13 +90,13 @@ init_conf() -> emqx_app:set_init_config_load_done(). cluster_nodes() -> - maps:get(running_nodes, ekka_cluster:info()) -- [node()]. + mria_mnesia:running_nodes() -- [node()]. copy_override_conf_from_core_node() -> case cluster_nodes() of %% The first core nodes is self. [] -> - ?SLOG(debug, #{msg => "skip_copy_overide_conf_from_core_node"}), + ?SLOG(debug, #{msg => "skip_copy_override_conf_from_core_node"}), {ok, ?DEFAULT_INIT_TXN_ID}; Nodes -> {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes), @@ -130,7 +130,7 @@ copy_override_conf_from_core_node() -> %% finish the boot sequence and load the %% config for other nodes to copy it. ?SLOG(info, #{ - msg => "skip_copy_overide_conf_from_core_node", + msg => "skip_copy_override_conf_from_core_node", loading_from_disk => true, nodes => Nodes, failed => Failed, @@ -142,7 +142,7 @@ copy_override_conf_from_core_node() -> Jitter = rand:uniform(2_000), Timeout = 10_000 + Jitter, ?SLOG(info, #{ - msg => "copy_overide_conf_from_core_node_retry", + msg => "copy_override_conf_from_core_node_retry", timeout => Timeout, nodes => Nodes, failed => Failed, @@ -155,7 +155,7 @@ copy_override_conf_from_core_node() -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, ?SLOG(debug, #{ - msg => "copy_overide_conf_from_core_node_success", + msg => "copy_override_conf_from_core_node_success", node => Node, cluster_override_conf_file => application:get_env( emqx, cluster_override_conf_file diff --git a/changes/ce/fix-10313.en.md b/changes/ce/fix-10313.en.md new file mode 100644 index 000000000..ca1a3b391 --- /dev/null +++ b/changes/ce/fix-10313.en.md @@ -0,0 +1,2 @@ +Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. +Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node. diff --git a/changes/ce/fix-10313.zh.md b/changes/ce/fix-10313.zh.md new file mode 100644 index 000000000..94f118ece --- /dev/null +++ b/changes/ce/fix-10313.zh.md @@ -0,0 +1,2 @@ +确保当 core 或 replicant 节点启动时，仅从 core 节点复制 `cluster-override.conf` 文件。 +此前按照节点启动时间排序时，core 节点可能从 replicant 节点复制该文件。 From ec1871ffdee91d117dfecfecd0ba30ddacbe7c86 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:18:03 -0300 Subject: [PATCH 26/50] test(janitor): catch each callback invocation --- apps/emqx/test/emqx_test_janitor.erl | 2 +- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index c9b297dc7..c3f82a3e1 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -65,7 +65,7 @@ terminate(_Reason, #{callbacks := Callbacks}) -> handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; handle_call(terminate, _From, State = #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> Fun() end, Callbacks), + lists:foreach(fun(Fun) -> catch Fun() end, Callbacks), {stop, normal, ok, State}; handle_call(_Req, _From, State) -> {reply, error, State}. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 02a4c3c3b..8664f9745 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,7 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(), + emqx_common_test_helpers:call_janitor(30_000), ok = snabbkaffe:stop(), ok end. From 0e5a22f500911080cebbcc011c48c0b1f2804e49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:27:06 -0300 Subject: [PATCH 27/50] test(cassandra): cap test container memory usage Default memory usage is excessive. --- .ci/docker-compose-file/cassandra/cassandra.yaml | 4 ++-- .ci/docker-compose-file/docker-compose-cassandra.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 968efe5f6..bc1bf3357 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -469,8 +469,8 @@ concurrent_materialized_view_writes: 32 # accepting writes when the limit is exceeded until a flush completes, # and will trigger a flush based on memtable_cleanup_threshold # If omitted, Cassandra will set both to 1/4 the size of the heap. -# memtable_heap_space_in_mb: 2048 -# memtable_offheap_space_in_mb: 2048 +memtable_heap_space_in_mb: 2048 +memtable_offheap_space_in_mb: 2048 # memtable_cleanup_threshold is deprecated. The default calculation # is the only reasonable choice. See the comments on memtable_flush_writers diff --git a/.ci/docker-compose-file/docker-compose-cassandra.yaml b/.ci/docker-compose-file/docker-compose-cassandra.yaml index a54f621c1..f7143f471 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra.yaml @@ -12,6 +12,8 @@ services: environment: CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4" CASSANDRA_RPC_ADDRESS: "0.0.0.0" + HEAP_NEWSIZE: "128M" + MAX_HEAP_SIZE: "2048M" volumes: - ./certs:/certs #ports: From c1cb5357e1df4b4d3c6cd8ca19813c46bd9bb9b9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:48:33 +0200 Subject: [PATCH 28/50] fix: enable schema check --- apps/emqx_modules/src/emqx_delayed_api.erl | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index d4e7e5b90..3499d83ca 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -52,7 +52,7 @@ -define(INVALID_NODE, 'INVALID_NODE'). api_spec() -> - emqx_dashboard_swagger:spec(?MODULE). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 4a9cb6723..fdc13f354 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, From 0efa9c7a110ba2fede4b3abeea784bf058e1fb35 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:48:52 +0200 Subject: [PATCH 29/50] fix: pretty format error responses --- apps/emqx_modules/src/emqx_delayed_api.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 3499d83ca..766d23d6b 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -202,9 +202,9 @@ delayed_message(get, #{bindings := #{node := NodeBin, msgid := HexId}}) -> {200, Message#{payload => base64:encode(Payload)}} end; {error, not_found} -> - {404, generate_http_code_map(not_found, Id)}; + {404, generate_http_code_map(not_found, HexId)}; {badrpc, _} -> - {400, generate_http_code_map(invalid_node, Id)} + {400, generate_http_code_map(invalid_node, NodeBin)} end end ); @@ -271,19 +271,19 @@ generate_http_code_map(id_schema_error, Id) -> #{ code => ?MESSAGE_ID_SCHEMA_ERROR, message => - iolist_to_binary(io_lib:format("Message ID ~p schema error", [Id])) + iolist_to_binary(io_lib:format("Message ID ~s schema error", [Id])) }; generate_http_code_map(not_found, Id) -> #{ code => ?MESSAGE_ID_NOT_FOUND, message => - iolist_to_binary(io_lib:format("Message ID ~p not found", [Id])) + iolist_to_binary(io_lib:format("Message ID ~s not found", [Id])) }; generate_http_code_map(invalid_node, Node) -> #{ code => ?INVALID_NODE, message => - iolist_to_binary(io_lib:format("The node name ~p is invalid", [Node])) + iolist_to_binary(io_lib:format("The node name ~s is invalid", [Node])) }. make_maybe(X, Error, Fun) -> From 5d722f8d465178a6fdcc60816458868cfa0da529 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:55:25 +0200 Subject: [PATCH 30/50] style: add changelog --- changes/ce/fix-10315.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10315.en.md diff --git a/changes/ce/fix-10315.en.md b/changes/ce/fix-10315.en.md new file mode 100644 index 000000000..67445252d --- /dev/null +++ b/changes/ce/fix-10315.en.md @@ -0,0 +1 @@ +Fix crash checking `limit` and `page` parameters in `/mqtt/delayed/messages` API call. From ed25ee6fecfdd49304e276b57d5763f91711b4f6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 3 Apr 2023 10:58:31 -0300 Subject: [PATCH 31/50] test(crl): fix flaky test (v5.0) --- apps/emqx/test/emqx_crl_cache_SUITE.erl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 7a61f7835..01f9c7172 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -884,7 +884,20 @@ t_revoked(Config) -> {port, 8883} ]), process_flag(trap_exit, true), - ?assertMatch({error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C)), + Res = emqtt:connect(C), + %% apparently, sometimes there's some race condition in + %% `emqtt_sock:ssl_upgrade' when it calls + %% `ssl:conetrolling_process' and a bad match happens at that + %% point. + case Res of + {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}} -> + ok; + {error, closed} -> + %% race condition? + ok; + _ -> + ct:fail("unexpected result: ~p", [Res]) + end, ok. t_revoke_then_refresh(Config) -> From e978d86c866d96fd68d38001fe9fc412a4f208d2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 13:41:13 +0200 Subject: [PATCH 32/50] chore: add doc_lift for authorization.sources doc_lift is to make the doc render application to lift this field to the root level and force the field's doc to refernec it instead of expanding the structs in a nested way --- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 943978519..2f8b26894 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index b15d4abd4..6630ed526 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -492,7 +492,9 @@ authz_fields() -> ?ARRAY(?UNION(UnionMemberSelector)), #{ default => [], - desc => ?DESC(sources) + desc => ?DESC(sources), + %% doc_lift is force a root level reference instead of nesting sub-structs + extra => #{doc_lift => true} } )} ]. From 2d6ca69ffb2724463d4fd284a37521ce9f243515 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 16:37:18 +0200 Subject: [PATCH 33/50] refactor: no support for listener level authentication for now --- apps/emqx/src/emqx_schema.erl | 4 +++- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0f90677bd..6bfff38d3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1880,7 +1880,9 @@ mqtt_listener(Bind) -> default => <<"3s">> } )}, - {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)} + {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, (authentication(listener))#{ + importance => ?IMPORTANCE_HIDDEN + }} ]. base_listener(Bind) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 741fb98ae..28c1e6f89 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -580,6 +580,8 @@ authentication_schema() -> #{ required => {false, recursively}, desc => ?DESC(gateway_common_authentication), + %% we do not expose this to the user for now + importance => ?IMPORTANCE_HIDDEN, examples => emqx_authn_api:authenticator_examples() } ). From a4e27e56a855389f60d42076353b2f4516f83bed Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 16:52:49 +0200 Subject: [PATCH 34/50] docs: add change logs --- changes/ce/fix-10317.en.md | 1 + changes/ce/fix-10317.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/fix-10317.en.md create mode 100644 changes/ce/fix-10317.zh.md diff --git a/changes/ce/fix-10317.en.md b/changes/ce/fix-10317.en.md new file mode 100644 index 000000000..7a83dcaca --- /dev/null +++ b/changes/ce/fix-10317.en.md @@ -0,0 +1 @@ +Do not expose listener level authentications before extensive verification. diff --git a/changes/ce/fix-10317.zh.md b/changes/ce/fix-10317.zh.md new file mode 100644 index 000000000..69cf09901 --- /dev/null +++ b/changes/ce/fix-10317.zh.md @@ -0,0 +1 @@ +在大量验证完成前不暴露监听器级的认证功能。 From 8b5a717a1f7617a25e2d1e6a132063441f7850a5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 11:02:15 -0300 Subject: [PATCH 35/50] test(peer): increase init and startup timeout for peer nodes Attempt to stabilize tests that use cluster nodes. --- apps/emqx/test/emqx_common_test_helpers.erl | 6 +++--- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 9a4461fac..077ebe138 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -660,6 +660,7 @@ start_slave(Name, Opts) when is_list(Opts) -> start_slave(Name, Opts) when is_map(Opts) -> SlaveMod = maps:get(peer_mod, Opts, ct_slave), Node = node_name(Name), + put_peer_mod(Node, SlaveMod), DoStart = fun() -> case SlaveMod of @@ -669,8 +670,8 @@ start_slave(Name, Opts) when is_map(Opts) -> [ {kill_if_fail, true}, {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, + {init_timeout, 20_000}, + {startup_timeout, 20_000}, {erl_flags, erl_flags()} ] ); @@ -687,7 +688,6 @@ start_slave(Name, Opts) when is_map(Opts) -> throw(Other) end, pong = net_adm:ping(Node), - put_peer_mod(Node, SlaveMod), setup_node(Node, Opts), ok = snabbkaffe:forward_trace(Node), Node. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 8664f9745..4019a9c42 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,9 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(30_000), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), ok = snabbkaffe:stop(), ok end. @@ -1664,7 +1666,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1875,7 +1877,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1894,10 +1896,14 @@ t_cluster_node_down(Config) -> {ok, _} = snabbkaffe:receive_events(SRef0), lists:foreach( fun(N) -> - ?assertMatch( - {ok, _}, - erpc:call(N, emqx_bridge, lookup, [BridgeId]), - #{node => N} + ?retry( + _Sleep1 = 100, + _Attempts1 = 50, + ?assertMatch( + {ok, _}, + erpc:call(N, emqx_bridge, lookup, [BridgeId]), + #{node => N} + ) ) end, Nodes From f3ffc02bff34a89f173b272ebea2abe8acd4bc0e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 09:41:25 -0300 Subject: [PATCH 36/50] feat(bridges): enable async query mode for all bridges with buffer workers Fixes https://emqx.atlassian.net/browse/EMQX-9130 Since buffer workers always support async calls ("outer calls"), we should decouple those two call modes (inner and outer), and avoid exposing the inner call configuration to user to avoid complexity. For bridges that currently only allow sync query modes, we should allow them to be configured with async. That means basically all bridge types except Kafka Producer. --- .../src/emqx_connector_mysql.erl | 19 +- .../src/emqx_connector_pgsql.erl | 3 +- .../src/schema/emqx_resource_schema.erl | 23 +-- .../test/emqx_rule_funcs_SUITE.erl | 4 - changes/ce/feat-10306.en.md | 3 + .../src/emqx_ee_bridge_cassa.erl | 15 +- .../src/emqx_ee_bridge_mongodb.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 15 +- .../src/emqx_ee_bridge_pgsql.erl | 15 +- .../src/emqx_ee_bridge_redis.erl | 4 +- .../src/emqx_ee_bridge_rocketmq.erl | 15 +- .../src/emqx_ee_bridge_tdengine.erl | 15 +- .../test/emqx_ee_bridge_cassa_SUITE.erl | 120 +++++++++-- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 94 +++++++-- .../test/emqx_ee_bridge_mysql_SUITE.erl | 190 ++++++++++++++---- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 189 +++++++++++++---- .../test/emqx_ee_bridge_redis_SUITE.erl | 22 +- .../test/emqx_ee_bridge_rocketmq_SUITE.erl | 12 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 156 +++++++++++--- .../src/emqx_ee_connector_mongodb.erl | 4 +- rel/i18n/emqx_resource_schema.hocon | 11 - 21 files changed, 639 insertions(+), 292 deletions(-) create mode 100644 changes/ce/feat-10306.en.md diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 68ec59894..fe495252a 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -172,10 +172,15 @@ on_query( %% not return result, next loop will try again on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, State); {error, Reason} -> - LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, - ?SLOG( + ?tp( error, - LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason} + "mysql_connector_do_prepare_failed", + #{ + connector => InstId, + sql => SQLOrKey, + state => State, + reason => Reason + } ), {error, Reason} end; @@ -417,12 +422,10 @@ on_sql_query( ), do_sql_query(SQLFunc, Conn, SQLOrKey, Params, Timeout, LogMeta); {error, disconnected} -> - ?SLOG( + ?tp( error, - LogMeta#{ - msg => "mysql_connector_do_sql_query_failed", - reason => worker_is_disconnected - } + "mysql_connector_do_sql_query_failed", + LogMeta#{reason => worker_is_disconnected} ), {error, {recoverable_error, disconnected}} end. diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 1fc994275..14cbbc80f 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -44,7 +44,8 @@ execute_batch/3 ]). --export([do_get_status/1]). +%% for ecpool workers usage +-export([do_get_status/1, prepare_sql_to_conn/2]). -define(PGSQL_HOST_OPTIONS, #{ default_port => ?PGSQL_DEFAULT_PORT diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index e89278e8c..7db886542 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -30,18 +30,6 @@ namespace() -> "resource_schema". roots() -> []. -fields("resource_opts_sync_only") -> - [ - {resource_opts, - mk( - ref(?MODULE, "creation_opts_sync_only"), - resource_opts_meta() - )} - ]; -fields("creation_opts_sync_only") -> - Fields = fields("creation_opts"), - QueryMod = {query_mode, fun query_mode_sync_only/1}, - lists:keyreplace(query_mode, 1, Fields, QueryMod); fields("resource_opts") -> [ {resource_opts, @@ -117,12 +105,6 @@ query_mode(default) -> async; query_mode(required) -> false; query_mode(_) -> undefined. -query_mode_sync_only(type) -> enum([sync]); -query_mode_sync_only(desc) -> ?DESC("query_mode_sync_only"); -query_mode_sync_only(default) -> sync; -query_mode_sync_only(required) -> false; -query_mode_sync_only(_) -> undefined. - request_timeout(type) -> hoconsc:union([infinity, emqx_schema:duration_ms()]); request_timeout(desc) -> ?DESC("request_timeout"); request_timeout(default) -> <<"15s">>; @@ -167,7 +149,4 @@ max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; max_queue_bytes(required) -> false; max_queue_bytes(_) -> undefined. -desc("creation_opts") -> - ?DESC("creation_opts"); -desc("creation_opts_sync_only") -> - ?DESC("creation_opts"). +desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 209332fe7..94adb3506 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -687,10 +687,6 @@ t_jq(_) -> got_timeout end, ConfigRootKey = emqx_rule_engine_schema:namespace(), - DefaultTimeOut = emqx_config:get([ - ConfigRootKey, - jq_function_default_timeout - ]), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) diff --git a/changes/ce/feat-10306.en.md b/changes/ce/feat-10306.en.md new file mode 100644 index 000000000..11754c5c0 --- /dev/null +++ b/changes/ce/feat-10306.en.md @@ -0,0 +1,3 @@ +Add support for `async` query mode for most bridges. + +Before this change, some bridges (Cassandra, MongoDB, MySQL, Postgres, Redis, RocketMQ, TDengine) were only allowed to be created with a `sync` query mode. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 12f86fcf7..78db8352a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -86,21 +86,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_ee_connector_cassa:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> fields("post", cassandra); fields("put") -> @@ -115,8 +104,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for Cassandra using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index bc450f39b..5dd3ef121 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -38,7 +38,7 @@ fields("config") -> {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})}, {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} - ] ++ emqx_resource_schema:fields("resource_opts_sync_only"); + ] ++ emqx_resource_schema:fields("resource_opts"); fields(mongodb_rs) -> emqx_connector_mongo:fields(rs) ++ fields("config"); fields(mongodb_sharded) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index eed4172ab..e5c9a5aab 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -79,21 +79,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -105,8 +94,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for MySQL using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 46132bd99..b5f0c3e62 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -81,21 +81,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_connector_pgsql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> fields("post", pgsql); fields("put") -> @@ -110,8 +99,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for PostgreSQL using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index fa6958b6d..1861b56ec 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -180,10 +180,10 @@ resource_fields(Type) -> resource_creation_fields("redis_cluster") -> % TODO % Cluster bridge is currently incompatible with batching. - Fields = emqx_resource_schema:fields("creation_opts_sync_only"), + Fields = emqx_resource_schema:fields("creation_opts"), lists:foldl(fun proplists:delete/2, Fields, [batch_size, batch_time, enable_batch]); resource_creation_fields(_) -> - emqx_resource_schema:fields("creation_opts_sync_only"). + emqx_resource_schema:fields("creation_opts"). desc("config") -> ?DESC("desc_config"); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 124e18069..78fd527d3 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -80,21 +80,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), required => false} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{<<"request_timeout">> => ?DEFFAULT_REQ_TIMEOUT}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_ee_connector_rocketmq:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -106,8 +95,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for RocketMQ using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index f031cbfbf..7a958d45f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -80,19 +80,8 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ emqx_ee_connector_tdengine:fields(config); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); + ] ++ emqx_resource_schema:fields("resource_opts") ++ emqx_ee_connector_tdengine:fields(config); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -104,8 +93,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for TDengine using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index d040000e2..f1ea6e930 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -73,15 +73,16 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + QueryModeGroups = [{group, async}, {group, sync}], + BatchingGroups = [ + %{group, with_batch}, + {group, without_batch} + ], [ - {tcp, [ - %{group, with_batch}, - {group, without_batch} - ]}, - {tls, [ - %{group, with_batch}, - {group, without_batch} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. @@ -93,7 +94,6 @@ init_per_group(tcp, Config) -> {cassa_host, Host}, {cassa_port, Port}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "cassa_tcp"} | Config ]; @@ -104,10 +104,13 @@ init_per_group(tls, Config) -> {cassa_host, Host}, {cassa_port, Port}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "cassa_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -139,14 +142,15 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - connect_and_clear_table(Config), ok = snabbkaffe:stop(), + connect_and_clear_table(Config), delete_bridge(Config), ok. @@ -171,6 +175,7 @@ common_init(Config0) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), emqx_mgmt_api_test_util:init_suite(), % Connect to cassnadra directly and create the table + catch connect_and_drop_table(Config0), connect_and_create_table(Config0), {Name, CassaConf} = cassa_config(BridgeType, Config0), Config = @@ -250,9 +255,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), - BridgeConfig = ?config(cassa_config, Config), + BridgeConfig0 = ?config(cassa_config, Config), + BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> @@ -288,6 +297,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(cassa_name, Config), + BridgeType = ?config(cassa_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) when is_reference(Ref) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_cassa(Config) -> Opts = #{ nodes => [{?config(cassa_host, Config), ?config(cassa_port, Config)}], @@ -546,15 +576,27 @@ t_write_failure(Config) -> % ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), + QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {query, <<"SELECT count(1) AS T FROM system.local">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of - true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of + true -> + ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> + ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) end, ok. @@ -565,22 +607,56 @@ t_missing_data(Config) -> ), %% emqx_ee_connector_cassa will send missed data as a `null` atom %% to ecql driver - Result = send_message(Config, #{}), + {_, {ok, Event}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( %% TODO: match error msgs - {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}}, - Result + #{ + result := + {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}} + }, + Event ), ok. t_bad_sql_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + EnableBatch = ?config(enable_batch, Config), + Name = ?config(cassa_name, Config), + ResourceId = emqx_bridge_resource:resource_id(cassandra, Name), ?assertMatch( {ok, _}, - create_bridge(Config) + create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ) ), Request = {query, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + case receive_result(Ref, 5_000) of + {ok, Res} -> + Res; + timeout -> + ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), + ct:fail("no response received") + end + end, + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 9850c9529..116dcc729 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -9,6 +9,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -16,9 +17,8 @@ all() -> [ - {group, rs}, - {group, sharded}, - {group, single} + {group, async}, + {group, sync} | (emqx_common_test_helpers:all(?MODULE) -- group_tests()) ]. @@ -31,12 +31,23 @@ group_tests() -> ]. groups() -> + TypeGroups = [ + {group, rs}, + {group, sharded}, + {group, single} + ], [ + {async, TypeGroups}, + {sync, TypeGroups}, {rs, group_tests()}, {sharded, group_tests()}, {single, group_tests()} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(Type = rs, Config) -> MongoHost = os:getenv("MONGO_RS_HOST", "mongo1"), MongoPort = list_to_integer(os:getenv("MONGO_RS_PORT", "27017")), @@ -44,7 +55,7 @@ init_per_group(Type = rs, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -63,7 +74,7 @@ init_per_group(Type = sharded, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -82,7 +93,7 @@ init_per_group(Type = single, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -99,6 +110,7 @@ end_per_group(_Type, _Config) -> ok. init_per_suite(Config) -> + emqx_common_test_helpers:clear_screen(), Config. end_per_suite(_Config) -> @@ -109,11 +121,13 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> catch clear_db(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> catch clear_db(Config), delete_bridge(Config), + snabbkaffe:stop(), ok. %%------------------------------------------------------------------------------ @@ -140,7 +154,8 @@ mongo_type_bin(sharded) -> mongo_type_bin(single) -> <<"mongodb_single">>. -mongo_config(MongoHost, MongoPort0, rs = Type) -> +mongo_config(MongoHost, MongoPort0, rs = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Servers = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -154,13 +169,19 @@ mongo_config(MongoHost, MongoPort0, rs = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Servers] + [ + Name, + Servers, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}; -mongo_config(MongoHost, MongoPort0, sharded = Type) -> +mongo_config(MongoHost, MongoPort0, sharded = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Servers = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -173,13 +194,19 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Servers] + [ + Name, + Servers, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}; -mongo_config(MongoHost, MongoPort0, single = Type) -> +mongo_config(MongoHost, MongoPort0, single = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Server = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -192,10 +219,15 @@ mongo_config(MongoHost, MongoPort0, single = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Server] + [ + Name, + Server, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}. @@ -248,7 +280,7 @@ find_all(Config) -> Name = ?config(mongo_name, Config), #{<<"collection">> := Collection} = ?config(mongo_config, Config), ResourceID = emqx_bridge_resource:resource_id(Type, Name), - emqx_resource:query(ResourceID, {find, Collection, #{}, #{}}). + emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}). send_message(Config, Payload) -> Name = ?config(mongo_name, Config), @@ -266,7 +298,12 @@ t_setup_via_config_and_publish(Config) -> create_bridge(Config) ), Val = erlang:unique_integer(), - ok = send_message(Config, #{key => Val}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"key">> := Val}]}, find_all(Config) @@ -286,7 +323,12 @@ t_setup_via_http_api_and_publish(Config) -> create_bridge_http(MongoConfig) ), Val = erlang:unique_integer(), - ok = send_message(Config, #{key => Val}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"key">> := Val}]}, find_all(Config) @@ -297,7 +339,12 @@ t_payload_template(Config) -> {ok, _} = create_bridge(Config, #{<<"payload_template">> => <<"{\"foo\": \"${clientid}\"}">>}), Val = erlang:unique_integer(), ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), - ok = send_message(Config, #{key => Val, clientid => ClientId}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val, clientid => ClientId}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"foo">> := ClientId}]}, find_all(Config) @@ -314,11 +361,16 @@ t_collection_template(Config) -> ), Val = erlang:unique_integer(), ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), - ok = send_message(Config, #{ - key => Val, - clientid => ClientId, - mycollectionvar => <<"mycol">> - }), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{ + key => Val, + clientid => ClientId, + mycollectionvar => <<"mycol">> + }), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"foo">> := ClientId}]}, find_all(Config) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 93e9e6fee..38e31c7ae 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -45,15 +45,16 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout, t_uninitialized_prepared_statement], + BatchingGroups = [ + {group, with_batch}, + {group, without_batch} + ], + QueryModeGroups = [{group, async}, {group, sync}], [ - {tcp, [ - {group, with_batch}, - {group, without_batch} - ]}, - {tls, [ - {group, with_batch}, - {group, without_batch} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. @@ -65,7 +66,6 @@ init_per_group(tcp, Config) -> {mysql_host, MysqlHost}, {mysql_port, MysqlPort}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "mysql_tcp"} | Config ]; @@ -76,10 +76,13 @@ init_per_group(tls, Config) -> {mysql_host, MysqlHost}, {mysql_port, MysqlPort}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "mysql_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{batch_size, 100} | Config0], common_init(Config); @@ -99,6 +102,7 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> + emqx_common_test_helpers:clear_screen(), Config. end_per_suite(_Config) -> @@ -109,6 +113,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -237,6 +242,25 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 500}). +query_resource_async(Config, Request) -> + Name = ?config(mysql_name, Config), + BridgeType = ?config(mysql_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + unprepare(Config, Key) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), @@ -409,17 +433,29 @@ t_write_failure(Config) -> Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( - emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - case QueryMode of - sync -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, + begin + %% for some unknown reason, `?wait_async_action' and `subscribe' + %% hang and timeout if called inside `with_failure', but the event + %% happens and is emitted after the test pid dies!? + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_nack}), + 2_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ); + async -> send_message(Config, SentData) - ); - async -> - send_message(Config, SentData) - end - end), + end, + ?assertMatch({ok, [#{result := {error, _}}]}, snabbkaffe:receive_events(SRef)), + ok + end), + ok + end, fun(Trace0) -> ct:pal("trace: ~p", [Trace0]), Trace = ?of_kind(buffer_worker_flush_nack, Trace0), @@ -443,27 +479,52 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), {ok, _} = create_bridge(Config), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, Timeout = 1000, + %% for some unknown reason, `?wait_async_action' and `subscribe' + %% hang and timeout if called inside `with_failure', but the event + %% happens and is emitted after the test pid dies!? + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_nack}), + 2 * Timeout + ), emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) - ) + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData, [], Timeout}) + ); + async -> + query_resource(Config, {send_message, SentData, [], Timeout}), + ok + end, + ok end), + ?assertMatch({ok, [#{result := {error, _}}]}, snabbkaffe:receive_events(SRef)), ok. t_simple_sql_query(Config) -> + QueryMode = ?config(query_mode, Config), + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, case IsBatch of true -> ?assertEqual({error, {unrecoverable_error, batch_select_not_implemented}}, Result); false -> ?assertEqual({ok, [<<"T">>], [[1]]}, Result) @@ -471,25 +532,37 @@ t_simple_sql_query(Config) -> ok. t_missing_data(Config) -> + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), - BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_ack}), + 2_000 + ), + send_message(Config, #{}), + {ok, [Event]} = snabbkaffe:receive_events(SRef), case IsBatch of true -> ?assertMatch( - {error, - {unrecoverable_error, - {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}}, - Result + #{ + result := + {error, + {unrecoverable_error, + {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}} + }, + Event ); false -> ?assertMatch( - {error, {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}}, - Result + #{ + result := + {error, + {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} + }, + Event ) end, ok. @@ -500,14 +573,22 @@ t_bad_sql_parameter(Config) -> create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), + {_, {ok, Event}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), BatchSize = ?config(batch_size, Config), IsBatch = BatchSize > 1, case IsBatch of true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); + ?assertMatch(#{result := {error, {unrecoverable_error, invalid_request}}}, Event); false -> - ?assertEqual({error, {unrecoverable_error, {invalid_params, [bad_parameter]}}}, Result) + ?assertMatch( + #{result := {error, {unrecoverable_error, {invalid_params, [bad_parameter]}}}}, + Event + ) end, ok. @@ -515,7 +596,12 @@ t_nasty_sql_string(Config) -> ?assertMatch({ok, _}, create_bridge(Config)), Payload = list_to_binary(lists:seq(0, 255)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, - Result = send_message(Config, Message), + {Result, {ok, _}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := mysql_connector_query_return}, + 1_000 + ), ?assertEqual(ok, Result), ?assertMatch( {ok, [<<"payload">>], [[Payload]]}, @@ -561,12 +647,22 @@ t_unprepared_statement_query(Config) -> create_bridge(Config) ), Request = {prepared_query, unprepared_query, []}, - Result = query_resource(Config, Request), + {_, {ok, Event}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), BatchSize = ?config(batch_size, Config), IsBatch = BatchSize > 1, case IsBatch of - true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> ?assertEqual({error, {unrecoverable_error, prepared_statement_invalid}}, Result) + true -> + ?assertMatch(#{result := {error, {unrecoverable_error, invalid_request}}}, Event); + false -> + ?assertMatch( + #{result := {error, {unrecoverable_error, prepared_statement_invalid}}}, + Event + ) end, ok. @@ -582,7 +678,13 @@ t_uninitialized_prepared_statement(Config) -> unprepare(Config, send_message), ?check_trace( begin - ?assertEqual(ok, send_message(Config, SentData)), + {Res, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := mysql_connector_query_return}, + 2_000 + ), + ?assertEqual(ok, Res), ok end, fun(Trace) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 10359a128..83cb8b1f3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -42,19 +42,18 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + BatchVariantGroups = [ + {group, with_batch}, + {group, without_batch}, + {group, matrix}, + {group, timescale} + ], + QueryModeGroups = [{async, BatchVariantGroups}, {sync, BatchVariantGroups}], [ - {tcp, [ - {group, with_batch}, - {group, without_batch}, - {group, matrix}, - {group, timescale} - ]}, - {tls, [ - {group, with_batch}, - {group, without_batch}, - {group, matrix}, - {group, timescale} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchVariantGroups}, + {sync, BatchVariantGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs}, {matrix, [t_setup_via_config_and_publish, t_setup_via_http_api_and_publish]}, @@ -68,7 +67,6 @@ init_per_group(tcp, Config) -> {pgsql_host, Host}, {pgsql_port, Port}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "pgsql_tcp"} | Config ]; @@ -79,10 +77,13 @@ init_per_group(tls, Config) -> {pgsql_host, Host}, {pgsql_port, Port}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "pgsql_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -118,6 +119,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -221,9 +223,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), - PGConfig = ?config(pgsql_config, Config), + PGConfig0 = ?config(pgsql_config, Config), + PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> @@ -251,6 +257,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(pgsql_name, Config), + BridgeType = ?config(pgsql_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_pgsql(Config) -> Opts = #{ host => ?config(pgsql_host, Config), @@ -308,11 +335,12 @@ t_setup_via_config_and_publish(Config) -> SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertEqual({ok, 1}, send_message(Config, SentData)), - #{?snk_kind := pgsql_connector_query_return}, - 10_000 - ), + {_, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), ?assertMatch( Val, connect_and_get_payload(Config) @@ -336,6 +364,7 @@ t_setup_via_http_api_and_publish(Config) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PgsqlConfig0 = ?config(pgsql_config, Config), + QueryMode = ?config(query_mode, Config), PgsqlConfig = PgsqlConfig0#{ <<"name">> => Name, <<"type">> => BridgeType @@ -348,11 +377,18 @@ t_setup_via_http_api_and_publish(Config) -> SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertEqual({ok, 1}, send_message(Config, SentData)), - #{?snk_kind := pgsql_connector_query_return}, - 10_000 - ), + {Res, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), + case QueryMode of + async -> + ok; + sync -> + ?assertEqual({ok, 1}, Res) + end, ?assertMatch( Val, connect_and_get_payload(Config) @@ -457,28 +493,71 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - Timeout = 1000, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) - ) - end), + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := call_query_enter}), + 2_000 + ), + Res0 = + emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + Res1 = + case QueryMode of + async -> + query_resource_async(Config, {send_message, SentData}); + sync -> + query_resource(Config, {send_message, SentData}) + end, + ?assertMatch({ok, [_]}, snabbkaffe:receive_events(SRef)), + Res1 + end), + case Res0 of + {_, Ref} when is_reference(Ref) -> + case receive_result(Ref, 15_000) of + {ok, Res} -> + ?assertMatch({error, {unrecoverable_error, _}}, Res); + timeout -> + ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), + ct:fail("no response received") + end; + _ -> + ?assertMatch({error, {resource_error, #{reason := timeout}}}, Res0) + end, ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), + QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of - true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> ?assertMatch({ok, _, [{1}]}, Result) + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of + true -> + ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> + ?assertMatch({ok, _, [{1}]}, Result) end, ok. @@ -487,21 +566,40 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), + {_, {ok, Event}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( - {error, {unrecoverable_error, {error, error, <<"23502">>, not_null_violation, _, _}}}, - Result + #{ + result := + {error, + {unrecoverable_error, {error, error, <<"23502">>, not_null_violation, _, _}}} + }, + Event ), ok. t_bad_sql_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> @@ -515,5 +613,10 @@ t_nasty_sql_string(Config) -> ?assertMatch({ok, _}, create_bridge(Config)), Payload = list_to_binary(lists:seq(1, 127)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, - ?assertEqual({ok, 1}, send_message(Config, Message)), + {_, {ok, _}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := pgsql_connector_query_return}, + 1_000 + ), ?assertEqual(Payload, connect_and_get_payload(Config)). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index 5431cbb03..f0b70d21b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -64,14 +64,17 @@ groups() -> {group, batch_on}, {group, batch_off} ], + QueryModeGroups = [{group, async}, {group, sync}], [ {rest, TCs}, {transports, [ {group, tcp}, {group, tls} ]}, - {tcp, TypeGroups}, - {tls, TypeGroups}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, TypeGroups}, + {sync, TypeGroups}, {redis_single, BatchGroups}, {redis_sentinel, BatchGroups}, {redis_cluster, BatchGroups}, @@ -79,6 +82,10 @@ groups() -> {batch_off, ResourceSpecificTCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(Group, Config) when Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster -> @@ -149,8 +156,9 @@ init_per_testcase(_Testcase, Config) -> {skip, "Batching is not supported by 'redis_cluster' bridge type"}; {RedisType, BatchMode} -> Transport = ?config(transport, Config), + QueryMode = ?config(query_mode, Config), #{RedisType := #{Transport := RedisConnConfig}} = redis_connect_configs(), - #{BatchMode := ResourceConfig} = resource_configs(), + #{BatchMode := ResourceConfig} = resource_configs(#{query_mode => QueryMode}), IsBatch = (BatchMode =:= batch_on), BridgeConfig0 = maps:merge(RedisConnConfig, ?COMMON_REDIS_OPTS), BridgeConfig1 = BridgeConfig0#{<<"resource_opts">> => ResourceConfig}, @@ -301,7 +309,7 @@ t_permanent_error(_Config) -> ?wait_async_action( publish_message(Topic, Payload), #{?snk_kind := redis_ee_connector_send_done}, - 10000 + 10_000 ) end, fun(Trace) -> @@ -529,14 +537,14 @@ invalid_command_bridge_config() -> <<"command_template">> => [<<"BAD">>, <<"COMMAND">>, <<"${payload}">>] }. -resource_configs() -> +resource_configs(#{query_mode := QueryMode}) -> #{ batch_off => #{ - <<"query_mode">> => <<"sync">>, + <<"query_mode">> => atom_to_binary(QueryMode), <<"start_timeout">> => <<"15s">> }, batch_on => #{ - <<"query_mode">> => <<"sync">>, + <<"query_mode">> => atom_to_binary(QueryMode), <<"worker_pool_size">> => <<"1">>, <<"batch_size">> => integer_to_binary(?BATCH_SIZE), <<"start_timeout">> => <<"15s">>, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index cd02b65d0..95ec47e7f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -24,17 +24,24 @@ all() -> [ - {group, with_batch}, - {group, without_batch} + {group, async}, + {group, sync} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), + BatchingGroups = [{group, with_batch}, {group, without_batch}], [ + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs}, {without_batch, TCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{batch_size, ?BATCH_SIZE} | Config0], common_init(Config); @@ -84,7 +91,6 @@ common_init(ConfigT) -> Config0 = [ {host, Host}, {port, Port}, - {query_mode, sync}, {proxy_name, "rocketmq"} | ConfigT ], diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index 3b580ec61..c956a93c6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -46,18 +46,25 @@ all() -> [ - {group, with_batch}, - {group, without_batch} + {group, async}, + {group, sync} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], [ + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -87,6 +94,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -109,7 +117,6 @@ common_init(ConfigT) -> Config0 = [ {td_host, Host}, {td_port, Port}, - {query_mode, sync}, {proxy_name, "tdengine_restful"} | ConfigT ], @@ -194,9 +201,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), - TDConfig = ?config(tdengine_config, Config), + TDConfig0 = ?config(tdengine_config, Config), + TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> @@ -224,6 +235,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(tdengine_name, Config), + BridgeType = ?config(tdengine_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_tdengine(Config) -> Opts = [ {host, to_bin(?config(td_host, Config))}, @@ -273,12 +305,14 @@ t_setup_via_config_and_publish(Config) -> SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertMatch( - {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 ), - #{?snk_kind := tdengine_connector_query_return}, - 10_000 + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, Result ), ?assertMatch( ?PAYLOAD, @@ -297,24 +331,32 @@ t_setup_via_config_and_publish(Config) -> t_setup_via_http_api_and_publish(Config) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), - PgsqlConfig0 = ?config(tdengine_config, Config), - PgsqlConfig = PgsqlConfig0#{ + QueryMode = ?config(query_mode, Config), + TDengineConfig0 = ?config(tdengine_config, Config), + TDengineConfig = TDengineConfig0#{ <<"name">> => Name, <<"type">> => BridgeType }, ?assertMatch( {ok, _}, - create_bridge_http(PgsqlConfig) + create_bridge_http(TDengineConfig) ), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertMatch( - {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) - ), - #{?snk_kind := tdengine_connector_query_return}, - 10_000 + Request = {send_message, SentData}, + Res0 = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, Res0 ), ?assertMatch( ?PAYLOAD, @@ -359,7 +401,14 @@ t_write_failure(Config) -> {ok, _} = create_bridge(Config), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch({error, econnrefused}, send_message(Config, SentData)) + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch({error, econnrefused}, Result), + ok end), ok. @@ -369,24 +418,50 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData}) - ) - end), + %% FIXME: TDengine connector hangs indefinetily during + %% `call_query' while the connection is unresponsive. Should add + %% a timeout to `APPLY_RESOURCE' in buffer worker?? + case QueryMode of + sync -> + emqx_common_test_helpers:with_failure( + timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData}) + ) + end + ); + async -> + ct:comment("tdengine connector hangs the buffer worker forever") + end, ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {query, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + {_, {ok, #{result := Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); false -> @@ -399,7 +474,12 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( {error, #{ <<"code">> := 534, @@ -410,13 +490,19 @@ t_missing_data(Config) -> ok. t_bad_sql_parameter(Config) -> + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + {_, {ok, #{result := Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> @@ -443,9 +529,15 @@ t_nasty_sql_string(Config) -> % [1]: https://github.com/taosdata/TDengine/blob/066cb34a/source/libs/parser/src/parUtil.c#L279-L301 Payload = list_to_binary(lists:seq(1, 127)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( {ok, #{<<"code">> := 0, <<"rows">> := 1}}, - send_message(Config, Message) + Result ), ?assertEqual( Payload, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index 8df77fbe0..aa03863b0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -60,7 +60,9 @@ on_query(InstanceId, {send_message, Message0}, State) -> collection => emqx_plugin_libs_rule:proc_tmpl(CollectionTemplate, Message0) }, Message = render_message(PayloadTemplate, Message0), - emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState); + Res = emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState), + ?tp(mongo_ee_connector_on_query_return, #{result => Res}), + Res; on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState). diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 933aa009b..c73f8b1aa 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -100,17 +100,6 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise } } - query_mode_sync_only { - desc { - en: """Query mode. Only support 'sync'.""" - zh: """请求模式。目前只支持同步模式。""" - } - label { - en: """Query mode""" - zh: """请求模式""" - } - } - request_timeout { desc { en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" From 2f2e8b8218a315560d3f7c1582a8b339da3bdbc2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 20:23:31 +0200 Subject: [PATCH 37/50] chore: bump version to e5.0.2-rc.3 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index d246b4639..b1cd5f98e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.21"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.2"). +-define(EMQX_RELEASE_EE, "5.0.2-rc.3"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From e1d589420933f55f528b9b2cccc07fda99a4593b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 20:28:29 +0200 Subject: [PATCH 38/50] docs: delete clickhous bridge doc fix change log because the feature is not released yet (new to e5.0.2) --- changes/ee/fix-10289.en.md | 1 - changes/ee/fix-10289.zh.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 changes/ee/fix-10289.en.md delete mode 100644 changes/ee/fix-10289.zh.md diff --git a/changes/ee/fix-10289.en.md b/changes/ee/fix-10289.en.md deleted file mode 100644 index 65eed7b5d..000000000 --- a/changes/ee/fix-10289.en.md +++ /dev/null @@ -1 +0,0 @@ -Clickhouse has got a fix that makes the error message better when users click the test button in the settings dialog. diff --git a/changes/ee/fix-10289.zh.md b/changes/ee/fix-10289.zh.md deleted file mode 100644 index d47278c16..000000000 --- a/changes/ee/fix-10289.zh.md +++ /dev/null @@ -1 +0,0 @@ -Clickhouse 已经修复了一个问题，当用户在设置对话框中点击测试按钮时，错误信息会更清晰。 From 11b3264251b9af73f1ac76b65096c28c265c6fbf Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:02:45 +0800 Subject: [PATCH 39/50] fix: redact the password to `******` in API examples fix #10222 --- apps/emqx_authn/src/emqx_authn_api.erl | 8 ++++---- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 0a7f67f5a..ad9cd8579 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1419,14 +1419,14 @@ request_user_create_examples() -> summary => <<"Regular user">>, value => #{ user_id => <<"user1">>, - password => <<"secret">> + password => <<"******">> } }, super_user => #{ summary => <<"Superuser">>, value => #{ user_id => <<"user2">>, - password => <<"secret">>, + password => <<"******">>, is_superuser => true } } @@ -1437,13 +1437,13 @@ request_user_update_examples() -> regular_user => #{ summary => <<"Update regular user">>, value => #{ - password => <<"newsecret">> + password => <<"******">> } }, super_user => #{ summary => <<"Update user and promote to superuser">>, value => #{ - password => <<"newsecret">>, + password => <<"******">>, is_superuser => true } } diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 586c66bef..44a478bca 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -235,7 +235,7 @@ mqtt_main_example() -> server => <<"127.0.0.1:1883">>, proto_ver => <<"v4">>, username => <<"foo">>, - password => <<"bar">>, + password => <<"******">>, clean_start => true, keepalive => <<"300s">>, retry_interval => <<"15s">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 80a317d2b..0b611c142 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -50,7 +50,7 @@ values(_Method, Type) -> database => <<"mqtt">>, pool_size => 8, username => <<"default">>, - password => <<"public">>, + password => <<"******">>, sql => ?DEFAULT_SQL, batch_value_separator => ?DEFAULT_BATCH_VALUE_SEPARATOR, local_topic => <<"local/topic/#">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e55be61e5..e6a3d1a58 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -46,7 +46,7 @@ values(_Method) -> database => <<"mqtt">>, pool_size => 8, username => <<"root">>, - password => <<"public">>, + password => <<"******">>, template => ?DEFAULT_TEMPLATE, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 1ad3af23c..5693a1902 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -61,7 +61,7 @@ values("influxdb_api_v1", post) -> TypeOpts = #{ database => <<"example_database">>, username => <<"example_username">>, - password => <<"examlpe_password">>, + password => <<"******">>, server => <<"127.0.0.1:8086">> }, values(common, "influxdb_api_v1", SupportUint, TypeOpts); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 529e28dea..f3dfa5964 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -64,7 +64,7 @@ values(common_config) -> authentication => #{ mechanism => <<"plain">>, username => <<"username">>, - password => <<"password">> + password => <<"******">> }, bootstrap_hosts => <<"localhost:9092">>, connect_timeout => <<"5s">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index bc450f39b..57763425a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -149,7 +149,7 @@ values(common, MongoType, Method, TypeOpts) -> srv_record => false, pool_size => 8, username => <<"myuser">>, - password => <<"mypass">> + password => <<"******">> }, MethodVals = method_values(MongoType, Method), Vals0 = maps:merge(MethodVals, Common), diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index eed4172ab..5825eb370 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -47,7 +47,7 @@ values(_Method) -> database => <<"test">>, pool_size => 8, username => <<"root">>, - password => <<"">>, + password => <<"******">>, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 46132bd99..2b292b4b6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -49,7 +49,7 @@ values(_Method, Type) -> database => <<"mqtt">>, pool_size => 8, username => <<"root">>, - password => <<"public">>, + password => <<"******">>, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index fa6958b6d..bcc2e9ba1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -77,7 +77,7 @@ values(common, RedisType, SpecificOpts) -> enable => true, local_topic => <<"local/topic/#">>, pool_size => 8, - password => <<"secret">>, + password => <<"******">>, command_template => [<<"LPUSH">>, <<"MSGS">>, <<"${payload}">>], resource_opts => values(resource_opts, RedisType, #{}), ssl => #{enable => false} From 6b2419998d29a708774c25ec0dbbe43e0a6476ae Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:06:23 +0800 Subject: [PATCH 40/50] chore: bump emqx_authn version --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 6a3ffbdb4..caa59e455 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, From b16c516e6b8eccf7deced1e36908965ba963e98c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 10:28:47 +0200 Subject: [PATCH 41/50] refactor: rename cluster_rpc_handler to cluster_rpc_cleaner this reflects what is actually does --- ...x_cluster_rpc_handler.erl => emqx_cluster_rpc_cleaner.erl} | 4 +++- apps/emqx_conf/src/emqx_conf_sup.erl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) rename apps/emqx_conf/src/{emqx_cluster_rpc_handler.erl => emqx_cluster_rpc_cleaner.erl} (97%) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl similarity index 97% rename from apps/emqx_conf/src/emqx_cluster_rpc_handler.erl rename to apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index c3d946a91..bce866c2d 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -13,7 +13,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_cluster_rpc_handler). + +%% @doc This module is responsible for cleaning up the cluster RPC MFA. +-module(emqx_cluster_rpc_cleaner). -behaviour(gen_server). diff --git a/apps/emqx_conf/src/emqx_conf_sup.erl b/apps/emqx_conf/src/emqx_conf_sup.erl index d4411af4b..6a3d795ae 100644 --- a/apps/emqx_conf/src/emqx_conf_sup.erl +++ b/apps/emqx_conf/src/emqx_conf_sup.erl @@ -36,7 +36,7 @@ init([]) -> ChildSpecs = [ child_spec(emqx_cluster_rpc, []), - child_spec(emqx_cluster_rpc_handler, []) + child_spec(emqx_cluster_rpc_cleaner, []) ], {ok, {SupFlags, ChildSpecs}}. From a7e5709f860c14572a4ef0c9b205f0a4a6de6107 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:22:02 +0800 Subject: [PATCH 42/50] chore: update changes --- changes/ce/fix-10323.en.md | 2 ++ changes/ce/fix-10323.zh.md | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 changes/ce/fix-10323.en.md create mode 100644 changes/ce/fix-10323.zh.md diff --git a/changes/ce/fix-10323.en.md b/changes/ce/fix-10323.en.md new file mode 100644 index 000000000..1bb678875 --- /dev/null +++ b/changes/ce/fix-10323.en.md @@ -0,0 +1,2 @@ +For security reasons, the value of the `password` field in the API examples is replaced with `******`. + diff --git a/changes/ce/fix-10323.zh.md b/changes/ce/fix-10323.zh.md new file mode 100644 index 000000000..4f7acfc56 --- /dev/null +++ b/changes/ce/fix-10323.zh.md @@ -0,0 +1,2 @@ +出于安全原因，将 API 示例中 `password` 字段的值，统一更换为 `******`。 + From 974b180da8a3670bc75bf9428822ae165c50f52c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 12:39:33 +0200 Subject: [PATCH 43/50] build: fix buildx.sh with git config --- scripts/buildx.sh | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 4f12e0abc..5c3a65369 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -27,6 +27,13 @@ help() { echo " E.g. ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11" } +die() { + msg="$1" + echo "$msg" >&2 + help + exit 1 +} + while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -81,13 +88,23 @@ while [ "$#" -gt 0 ]; do esac done -if [ -z "${PROFILE:-}" ] || - [ -z "${PKGTYPE:-}" ] || - [ -z "${BUILDER:-}" ] || - [ -z "${ARCH:-}" ]; then - help - exit 1 +## we have a different naming for them +if [[ $(uname -m) == "x86_64" ]]; then + NATIVE_ARCH='amd64' +elif [[ $(uname -m) == "aarch64" ]]; then + NATIVE_ARCH='arm64' +elif [[ $(uname -m) == "arm64" ]]; then + NATIVE_ARCH='arm64' +elif [[ $(uname -m) == "armv7l" ]]; then + # CHECKME: really ? + NATIVE_ARCH='arm64' fi +ARCH="${ARCH:-${NATIVE_ARCH:-}}" + +[ -z "${PROFILE:-}" ] && die "missing --prifile" +[ -z "${PKGTYPE:-}" ] && die "missing --pkgtyp" +[ -z "${BUILDER:-}" ] && die "missing --builder" +[ -z "${ARCH:-}" ] && die "missing --arch" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -128,13 +145,7 @@ if [[ "$HOST_SYSTEM" = "$BUILDER_SYSTEM" ]]; then fi IS_NATIVE_ARCH='no' -if [[ $(uname -m) == "x86_64" && "$ARCH" == "amd64" ]]; then - IS_NATIVE_ARCH='yes' -elif [[ $(uname -m) == "aarch64" && "$ARCH" == "arm64" ]]; then - IS_NATIVE_ARCH='yes' -elif [[ $(uname -m) == "arm64" && "$ARCH" == "arm64" ]]; then - IS_NATIVE_ARCH='yes' -elif [[ $(uname -m) == "armv7l" && "$ARCH" == "arm64" ]]; then +if [[ "$NATIVE_ARCH" == "$ARCH" ]]; then IS_NATIVE_ARCH='yes' fi @@ -151,7 +162,7 @@ elif docker info; then --platform="linux/$ARCH" \ --env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \ "$BUILDER" \ - bash -euc "$CMD_RUN" + bash -euc "git config --global --add safe.directory /emqx && $CMD_RUN" else echo "Error: Docker not available on unsupported platform" exit 1; From 8fd9dd741e4eaff402398b0e93aaf8336c47e584 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 12:39:55 +0200 Subject: [PATCH 44/50] fix(emqx_conf_app): wait for tables ready beofre starting apps --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 38 +++++++++++++++++++++++-- apps/emqx_conf/src/emqx_conf_app.erl | 3 ++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 89f678554..d03b57b03 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -270,9 +270,6 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> - %% Workaround for https://github.com/emqx/mria/issues/94: - _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), - _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, TnxId = emqx_app:get_init_tnx_id(), @@ -281,6 +278,7 @@ init([Node, RetryMs]) -> %% @private handle_continue(?CATCH_UP, State) -> + ok = wait_for_emqx_ready(), {noreply, State, catch_up(State)}. handle_call(reset, _From, State) -> @@ -566,3 +564,37 @@ maybe_init_tnx_id(_Node, TnxId) when TnxId < 0 -> ok; maybe_init_tnx_id(Node, TnxId) -> {atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]), ok. + +%% @priv Cannot proceed until emqx app is ready. +%% Otherwise the committed transaction catch up may fail. +wait_for_emqx_ready() -> + %% wait 10 seconds for emqx to start + ok = do_wait_for_emqx_ready(10). + +%% Wait for emqx app to be ready, +%% write a log message every 1 second +do_wait_for_emqx_ready(0) -> + timeout; +do_wait_for_emqx_ready(N) -> + %% check interval is 100ms + %% makes the total wait time 1 second + case do_wait_for_emqx_ready2(10) of + ok -> + ok; + timeout -> + ?SLOG(warning, #{msg => "stil_waiting_for_emqx_app_to_be_ready"}), + do_wait_for_emqx_ready(N - 1) + end. + +%% Wait for emqx app to be ready, +%% check interval is 100ms +do_wait_for_emqx_ready2(0) -> + timeout; +do_wait_for_emqx_ready2(N) -> + case emqx:is_running() of + true -> + ok; + false -> + timer:sleep(100), + do_wait_for_emqx_ready2(N - 1) + end. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index af42f0e1a..e342436dc 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -84,6 +84,9 @@ init_load() -> -endif. init_conf() -> + %% Workaround for https://github.com/emqx/mria/issues/94: + _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), + _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, TnxId} = copy_override_conf_from_core_node(), emqx_app:set_init_tnx_id(TnxId), init_load(), From 0b6fd7fe145aaee2c39ae12998c6706cf5e3daec Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 13:33:52 -0300 Subject: [PATCH 45/50] fix(buffer_worker): check request timeout and health check interval Port of https://github.com/emqx/emqx/pull/10154 for `release-50` Fixes https://emqx.atlassian.net/browse/EMQX-9099 Originally, the `resume_interval`, which is what defines how often a buffer worker will attempt to retry its inflight window, was set to the same as the `health_check_interval`. This had the problem that, with default values, `health_check_interval = request_timeout`. This meant that, if a buffer worker with those configs were ever blocked, all requests would have timed out by the time it retried them. Here we change the default `resume_interval` to a reasonable value dependent on `health_check_interval` and `request_timeout`, and also expose that as a hidden parameter for fine tuning if necessary. --- .../i18n/emqx_resource_schema_i18n.conf | 11 +++++++++++ .../src/emqx_resource_buffer_worker.erl | 17 ++++++++++++++++- .../src/schema/emqx_resource_schema.erl | 7 +++++++ changes/ce/fix-10154.en.md | 8 ++++++++ .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 14 ++++++++++++-- 5 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10154.en.md diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 600289b1d..57b109497 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -45,6 +45,17 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise } } + resume_interval { + desc { + en: """The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" + zh: """在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" + } + label { + en: """Resume Interval""" + zh: """重试时间间隔""" + } + } + start_after_created { desc { en: """Whether start the resource right after created.""" diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8bfd77e61..648587c25 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -88,6 +88,8 @@ -type queue_query() :: ?QUERY(reply_fun(), request(), HasBeenSent :: boolean(), expire_at()). -type request() :: term(). -type request_from() :: undefined | gen_statem:from(). +-type request_timeout() :: infinity | timer:time(). +-type health_check_interval() :: timer:time(). -type state() :: blocked | running. -type inflight_key() :: integer(). -type data() :: #{ @@ -199,6 +201,8 @@ init({Id, Index, Opts}) -> RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0), + DefaultResumeInterval = default_resume_interval(RequestTimeout, HealthCheckInterval), + ResumeInterval = maps:get(resume_interval, Opts, DefaultResumeInterval), Data = #{ id => Id, index => Index, @@ -207,7 +211,7 @@ init({Id, Index, Opts}) -> batch_size => BatchSize, batch_time => BatchTime, queue => Queue, - resume_interval => maps:get(resume_interval, Opts, HealthCheckInterval), + resume_interval => ResumeInterval, tref => undefined }, ?tp(buffer_worker_init, #{id => Id, index => Index}), @@ -1679,6 +1683,17 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +%% The request timeout should be greater than the resume interval, as +%% it defines how often the buffer worker tries to unblock. If request +%% timeout is <= resume interval and the buffer worker is ever +%% blocked, than all queued requests will basically fail without being +%% attempted. +-spec default_resume_interval(request_timeout(), health_check_interval()) -> timer:time(). +default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) -> + max(1, HealthCheckInterval); +default_resume_interval(RequestTimeout, HealthCheckInterval) -> + max(1, min(HealthCheckInterval, RequestTimeout div 3)). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). adjust_batch_time_test_() -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index fdd65bc3c..b9ed176fe 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -55,6 +55,7 @@ fields("creation_opts") -> [ {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, + {resume_interval, fun resume_interval/1}, {start_after_created, fun start_after_created/1}, {start_timeout, fun start_timeout/1}, {auto_restart_interval, fun auto_restart_interval/1}, @@ -81,6 +82,12 @@ worker_pool_size(default) -> ?WORKER_POOL_SIZE; worker_pool_size(required) -> false; worker_pool_size(_) -> undefined. +resume_interval(type) -> emqx_schema:duration_ms(); +resume_interval(hidden) -> true; +resume_interval(desc) -> ?DESC("resume_interval"); +resume_interval(required) -> false; +resume_interval(_) -> undefined. + health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; diff --git a/changes/ce/fix-10154.en.md b/changes/ce/fix-10154.en.md new file mode 100644 index 000000000..24bc4bae1 --- /dev/null +++ b/changes/ce/fix-10154.en.md @@ -0,0 +1,8 @@ +Change the default `resume_interval` for bridges and connectors to be +the minimum of `health_check_interval` and `request_timeout / 3`. +Also exposes it as a hidden configuration to allow fine tuning. + +Before this change, the default values for `resume_interval` meant +that, if a buffer ever got blocked due to resource errors or high +message volumes, then, by the time the buffer would try to resume its +normal operations, almost all requests would have timed out. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 8424ddff0..f9968ee96 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -520,6 +520,7 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> #{measurements := #{gauge_set := ExpectedValue}} -> ok; #{measurements := #{gauge_set := Value}} -> + ct:pal("events: ~p", [Events]), ct:fail( "gauge ~p didn't reach expected value ~p; last value: ~p", [GaugeName, ExpectedValue, Value] @@ -972,7 +973,13 @@ t_publish_econnrefused(Config) -> ResourceId = ?config(resource_id, Config), %% set pipelining to 1 so that one of the 2 requests is `pending' %% in ehttpc. - {ok, _} = create_bridge(Config, #{<<"pipelining">> => 1}), + {ok, _} = create_bridge( + Config, + #{ + <<"pipelining">> => 1, + <<"resource_opts">> => #{<<"resume_interval">> => <<"15s">>} + } + ), {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), assert_empty_metrics(ResourceId), @@ -986,7 +993,10 @@ t_publish_timeout(Config) -> %% requests are done separately. {ok, _} = create_bridge(Config, #{ <<"pipelining">> => 1, - <<"resource_opts">> => #{<<"batch_size">> => 1} + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"resume_interval">> => <<"15s">> + } }), {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), From 196ca43fbb88afa78c246ce5e8ea048faafa567f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 14:19:48 +0200 Subject: [PATCH 46/50] fix(emqx_conf_app): call the right API to retrieve core nodes --- apps/emqx_conf/src/emqx_conf_app.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index e342436dc..b88fa1947 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -93,7 +93,7 @@ init_conf() -> emqx_app:set_init_config_load_done(). cluster_nodes() -> - mria_mnesia:running_nodes() -- [node()]. + mria:cluster_nodes(cores) -- [node()]. copy_override_conf_from_core_node() -> case cluster_nodes() of From aca65ca2d495152d3809ed4d9ba0f33a18e1cae4 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 4 Apr 2023 15:27:29 +0300 Subject: [PATCH 47/50] fix(rule_engine): don't increment unknown counter on unrecoverable errors Closes: EMQX-8786 --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 6 +++--- changes/ce/fix-10327.en.md | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10327.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index ed6cd22de..153832246 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -514,6 +514,8 @@ inc_action_metrics({error, {recoverable_error, _}}, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); +inc_action_metrics({error, {unrecoverable_error, _}}, RuleId) -> + emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'); inc_action_metrics(R, RuleId) -> case is_ok_result(R) of false -> @@ -523,9 +525,7 @@ inc_action_metrics(R, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success') end. -is_ok_result(ok) -> - true; is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); -is_ok_result(ok) -> +is_ok_result(_) -> false. diff --git a/changes/ce/fix-10327.en.md b/changes/ce/fix-10327.en.md new file mode 100644 index 000000000..4fa561779 --- /dev/null +++ b/changes/ce/fix-10327.en.md @@ -0,0 +1,4 @@ +Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors. +This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown'). +The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges +are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success'). From 5925ff07c271a9dd108eb46cdceff2b422c01d1d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 15:20:30 +0200 Subject: [PATCH 48/50] test(emqx_cluster_rpc): fix test cases --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 ++ .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index d03b57b03..92c8794cd 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -278,6 +278,8 @@ init([Node, RetryMs]) -> %% @private handle_continue(?CATCH_UP, State) -> + %% emqx app must be started before + %% trying to catch up the rpc commit logs ok = wait_for_emqx_ready(), {noreply, State, catch_up(State)}. diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index f7d3c76fd..8cdfcaeea 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -43,6 +43,7 @@ groups() -> []. init_per_suite(Config) -> application:load(emqx_conf), ok = ekka:start(), + ok = emqx_common_test_helpers:start_apps([]), ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), ok = emqx_config:put([node, cluster_call, retry_interval], 1000), meck:new(emqx_alarm, [non_strict, passthrough, no_link]), @@ -53,6 +54,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), ekka:stop(), mria:stop(), meck:unload(mria), @@ -255,13 +257,13 @@ t_fast_forward_commit(_Config) -> ), ok. -t_handler_unexpected_msg(_Config) -> - Handler = emqx_cluster_rpc_handler, - OldPid = erlang:whereis(Handler), - ok = gen_server:cast(Handler, unexpected_cast_msg), - ignore = gen_server:call(Handler, unexpected_cast_msg), - erlang:send(Handler, unexpected_info_msg), - NewPid = erlang:whereis(Handler), +t_cleaner_unexpected_msg(_Config) -> + Cleaner = emqx_cluster_cleaner, + OldPid = erlang:whereis(Cleaner), + ok = gen_server:cast(Cleaner, unexpected_cast_msg), + ignore = gen_server:call(Cleaner, unexpected_cast_msg), + erlang:send(Cleaner, unexpected_info_msg), + NewPid = erlang:whereis(Cleaner), ?assertEqual(OldPid, NewPid), ok. @@ -279,8 +281,8 @@ start() -> {ok, Pid1} = emqx_cluster_rpc:start_link(), {ok, Pid2} = emqx_cluster_rpc:start_link({node(), ?NODE2}, ?NODE2, 500), {ok, Pid3} = emqx_cluster_rpc:start_link({node(), ?NODE3}, ?NODE3, 500), - {ok, Pid4} = emqx_cluster_rpc_handler:start_link(100, 500), - true = erlang:register(emqx_cluster_rpc_handler, Pid4), + {ok, Pid4} = emqx_cluster_rpc_cleaner:start_link(100, 500), + true = erlang:register(emqx_cluster_rpc_cleaner, Pid4), {ok, [Pid1, Pid2, Pid3, Pid4]}. stop() -> @@ -296,7 +298,7 @@ stop() -> end || N <- [?NODE1, ?NODE2, ?NODE3] ], - gen_server:stop(emqx_cluster_rpc_handler, normal, 5000). + gen_server:stop(emqx_cluster_rpc_cleaner, normal, 5000). receive_msg(0, _Msg) -> ok; From d25db3ace4b90fdcd19b86c54ba6780b14e30da4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Apr 2023 16:38:25 +0200 Subject: [PATCH 49/50] chore: bump version to e5.0.2-rc.4 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index b1cd5f98e..76920928b 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.21"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.3"). +-define(EMQX_RELEASE_EE, "5.0.2-rc.4"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 9810c9f7e3f2d92c84eeafc535f91fbe39434922 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 4 Apr 2023 17:36:51 +0300 Subject: [PATCH 50/50] refactor(rule_engine): test 'ok' results in one dedicated function --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 153832246..e8d807d38 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -508,8 +508,6 @@ nested_put(Alias, Val, Columns0) -> emqx_rule_maps:nested_put(Alias, Val, Columns). -define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found). -inc_action_metrics(ok, RuleId) -> - emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success'); inc_action_metrics({error, {recoverable_error, _}}, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) -> @@ -525,6 +523,8 @@ inc_action_metrics(R, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success') end. +is_ok_result(ok) -> + true; is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) ->